add some undocumented client commands from PC, GC and BB

Update comments for revive to confirm reverser/moon atomizer use same subcommands.

Added new subcommand A1 which is part of revive process. Unsure of function (cannot find any in game effects from it).

.github/workflows/cmake.yml

@@ -16,6 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
+        with_resource_file: ['true', 'false']
 
     steps:
     - uses: actions/checkout@v2
@@ -36,6 +37,15 @@ jobs:
         make
         sudo make install
 
+    - name: Install resource_file
+      if: ${{ matrix.with_resource_file == 'true' }}
+      run: |
+        git clone https://github.com/fuzziqersoftware/resource_dasm.git
+        cd resource_dasm
+        cmake .
+        make
+        sudo make install
+
     - name: Configure CMake
       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 

.gitignore

@@ -12,3 +12,10 @@ CTestTestFile.cmake
 Testing
 cmake_install.cmake
 install_manifest.txt
+
+# Files modified by the user and/or server that don't have defaults
+system/config.json
+system/licenses.nsi
+system/players/player_*
+system/players/account_*
+system/players/bank_*

CMakeLists.txt

@@ -21,7 +21,7 @@ set(CMAKE_BUILD_TYPE Debug)
 
 
 
-# Executable definitions
+# Library search
 
 find_path     (LIBEVENT_INCLUDE_DIR NAMES event.h)
 find_library  (LIBEVENT_LIBRARY     NAMES event)
@@ -31,12 +31,28 @@ set (LIBEVENT_LIBRARIES
         ${LIBEVENT_LIBRARY}
         ${LIBEVENT_CORE})
 
+find_path     (RESOURCE_FILE_INCLUDE_DIR NAMES resource_file/ResourceFile.hh)
+find_library  (RESOURCE_FILE_LIBRARY     NAMES resource_file)
+
+if(RESOURCE_FILE_INCLUDE_DIR AND RESOURCE_FILE_LIBRARY)
+  set(RESOURCE_FILE_FOUND 1)
+else()
+  set(RESOURCE_FILE_FOUND 0)
+endif()
+
+
+
+# Executable definition
+
 add_executable(newserv
+  src/Channel.cc
   src/ChatCommands.cc
   src/Client.cc
   src/Compression.cc
   src/DNSServer.cc
+  src/Episode3.cc
   src/FileContentsCache.cc
+  src/FunctionCompiler.cc
   src/IPFrameInfo.cc
   src/IPStackSimulator.cc
   src/Items.cc
@@ -68,6 +84,15 @@ add_executable(newserv
 target_include_directories(newserv PUBLIC ${LIBEVENT_INCLUDE_DIR})
 target_link_libraries(newserv phosg ${LIBEVENT_LIBRARIES})
 
+if(RESOURCE_FILE_FOUND)
+  target_compile_definitions(newserv PUBLIC HAVE_RESOURCE_FILE)
+  target_include_directories(newserv PUBLIC ${RESOURCE_FILE_INCLUDE_DIR})
+  target_link_libraries(newserv ${RESOURCE_FILE_LIBRARY})
+  message(STATUS "libresource_file found; enabling patch support")
+else()
+  message(WARNING "libresource_file not available; disabling patch support")
+endif()
+
 
 
 # Installation configuration

README.md

@@ -4,7 +4,7 @@ newserv is a game server and proxy for Phantasy Star Online (PSO).
 
 This project includes code that was reverse-engineered by the community in ages long past, and has been included in many projects since then. It also includes some game data from Phantasy Star Online itself; this data was originally created by Sega.
 
-This project is a rewrite of a rewrite of a game server that I wrote many years ago. So far, it works well with PSO GC Episodes 1 & 2, and lobbies (but not games) are implemented on Episode 3. Some basic functionality works on PSO PC and PSO BB, but there are probably still some cases that lead to errors (which will disconnect the client).
+This project is a rewrite of a rewrite of a game server that I wrote many years ago. So far, it works well with PSO GC Episodes 1 & 2, and lobbies (but not games) are implemented on Episode 3. Some basic functionality works on PSO PC and PSO BB, but there are probably still some cases that lead to errors (which will disconnect the client). The proxy works well with PSO GC and PSO BB.
 
 Feel free to submit GitHub issues if you find bugs or have feature requests. I'd like to make the server as stable and complete as possible, but I can't promise that I'll respond to issues in a timely manner.
 
@@ -16,6 +16,9 @@ Current known issues / missing features:
 - Test all the communication features (info board, simple mail, card search, etc.)
 - The trade window isn't implemented yet.
 - PSO PC and PSOBB are not well-tested and likely will disconnect when clients try to use unimplemented features. Only GC is known to be stable and mostly complete.
+- Patches currently are platform-specific but not version-specific. This makes them quite a bit harder to use properly.
+- Find a way to silence audio in RunDOL.s. Some old DOLs don't reset audio systems at load time and it's annoying to hear the crash buzz when the GC hasn't actually crashed.
+- Implement private lobbies, and add a way to make games persistent.
 
 ## Usage
 
@@ -24,22 +27,19 @@ Currently this code should build on macOS and Ubuntu. It will likely work on oth
 There is a probably-not-too-old macOS release on the newserv GitHub repository (look in the right sidebar).
 
 If you're running Linux or want to build newserv yourself, here's what you do:
-- Make sure you have CMake and libevent installed. (`brew install libevent` on macOS, `sudo apt-get install cmake libevent-dev` on most Linuxes)
-- Build and install phosg (https://github.com/fuzziqersoftware/phosg).
-- Run `cmake . && make` on the newserv directory.
-- In the system/ directory, make a copy of config.example.json named config.json, and edit it appropriately.
-- Run `./newserv` in the newserv directory. This will start the game server and run the interactive shell. You may need `sudo` if newserv's built-in DNS server is enabled.
-- Use the interactive shell to add a license. Run `help` in the shell to see how to do this.
+1. Make sure you have CMake and libevent installed. (`brew install cmake libevent` on macOS, `sudo apt-get install cmake libevent-dev` on most Linuxes)
+2. Build and install phosg (https://github.com/fuzziqersoftware/phosg).
+3. Optionally, install resource_dasm (https://github.com/fuzziqersoftware/resource_dasm). This will enable newserv to load DOL files on PSO GC clients. PSO GC clients can play PSO normally on newserv without this.
+4. Run `cmake . && make` on the newserv directory.
+
+After building newserv or downloading a release, do this to set it up and use it:
+1. In the system/ directory, make a copy of config.example.json named config.json, and edit it appropriately.
+2. Run `./newserv` in the newserv directory. This will start the game server and run the interactive shell. You may need `sudo` if newserv's built-in DNS server is enabled.
+3. Use the interactive shell to add a license. Run `help` in the shell to see how to do this.
 
 ### Installing quests
 
-newserv automatically finds quests in the system/quests directory. To install your own quests, or to use quests you've saved using the proxy's set-save-files option, just put them in that directory and name them appropriately.
-
-There are multiple PSO quest formats out there; newserv supports most of them. Specifically, newserv can use quests in any of the following formats:
-- bin/dat format: These quests consist of two files with the same base name, a .bin file and a .dat file.
-- Unencrypted GCI format: These quests also consist of a .bin and .dat file, but an encoding is applied on top of them. The filenames should end in .bin.gci and .dat.gci. (Note that there also exists an encrypted GCI format, which newserv does not support.)
-- Encrypted DLQ format: These quests also consist of a .bin and .dat file, but downlaod quest encryption is applied on top of them. The filenames should end in .bin.dlq and .dat.dlq.
-- QST format: These quests consist of only a .qst file, which contains both the .bin and .dat files within it.
+newserv automatically finds quests in the system/quests/ directory. To install your own quests, or to use quests you've saved using the proxy's set-save-files option, just put them in that directory and name them appropriately.
 
 Standard quest file names should be like `q###-CATEGORY-VERSION.EXT`; battle quests should be named like `b###-VERSION.EXT`, and challenge quests should be named like `c###-VERSION.EXT`. The fields in each filename are:
 - `###`: quest number (this doesn't really matter; it should just be unique for the version)
@@ -47,53 +47,72 @@ Standard quest file names should be like `q###-CATEGORY-VERSION.EXT`; battle que
 - `VERSION`: d1 = Dreamcast v1, dc = Dreamcast v2, pc = PC, gc = GameCube Episodes 1 & 2, gc3 = Episode 3, bb = Blue Burst
 - `EXT`: file extension (bin, dat, bin.gci, dat.gci, bin.dlq, dat.dlq, or qst)
 
+There are multiple PSO quest formats out there; newserv supports most of them. Specifically, newserv can use quests in any of the following formats:
+- bin/dat format: These quests consist of two files with the same base name, a .bin file and a .dat file.
+- Unencrypted GCI format: These quests also consist of a .bin and .dat file, but an encoding is applied on top of them. The filenames should end in .bin.gci and .dat.gci. (Note that there also exists an encrypted GCI format, which newserv does not support.)
+- Encrypted DLQ format: These quests also consist of a .bin and .dat file, but download quest encryption is applied on top of them. The filenames should end in .bin.dlq and .dat.dlq.
+- QST format: These quests consist of only a .qst file, which contains both the .bin and .dat files within it.
+
 When newserv indexes the quests during startup, it will warn (but not fail) if any quests are corrupt or in unrecognized formats.
 
 If you've changed the contents of the quests directory, you can re-index the quests without restarting the server by running `reload quests` in the interactive shell.
 
 All quests, including those originally in GCI or DLQ format, are treated as online quests unless their filenames specify the dl category. newserv allows players to download all quests, even those in non-download categories.
 
+### Patches and DOL files
+
+Everything in this section requires resource_dasm to be installed, so newserv can use the PowerPC assembler and disassembler from its libresource_file library. If resource_dasm is not installed, newserv will still build and run, but these features will not be available.
+
+You can put patches in the system/ppc directory with filenames like PatchName.patch.s and they will appear in the Patches menu for PSO GC clients that support patching. Patches are written in PowerPC assembly and are compiled when newserv is started. See system/ppc/WriteMemory.s for a commented example of such a function.
+
+You can also put DOL files in the system/dol directory, and they will appear in the Programs menu. Selecting a DOL file there will load the file into their GameCube's memory and run it, just like the old homebrew loaders (PSUL and PSOload) did. For this to work, ReadMemoryWord.s, WriteMemory.s, and RunDOL.s must be present in the system/ppc directory. This has been tested on Dolphin but not on a real GameCube, so results may vary.
+
+I mainly built the DOL loading functionality for documentation purposes. By now, there are many better ways to load homebrew code on an unmodified GameCube, but to my knowledge there isn't another open-source implementation of this method in existence.
+
 ### Chat commands
 
 The server's shell supports a variety of administration commands. If the interactive shell is enabled, you can enter these commands at any time, even if the prompt isn't visible. Run `help` in the server's shell to see all of the commands and how to use them.
 
-newserv also supports a variety of commands players can use via the chat interface. These commands work on the game server (that is, in lobbies and games hosted by newserv); they do not work on the proxy server. The chat commands are:
+newserv also supports a variety of commands players can use via the chat interface. Any chat message that begins with `$` is treated as a chat command. (If you actually want to send a chat message starting with `$`, type `$$` instead.)
+
+Some commands only work on the game server and not on the proxy server. The chat commands are:
 
 * Information commands
-    * `$li`: Shows basic information about the lobby or game you're in.
-    * `$what`: Shows the type, name, and stats of the nearest item on the ground.
+    * `$li`: Shows basic information about the lobby or game you're in. If you're on the proxy server, shows information about your connection (remote Guild Card number, client ID, etc.) instead.
+    * `$what` (game server only): Shows the type, name, and stats of the nearest item on the ground.
 
 * Personal state commands
     * `$arrow <color-id>`: Changes your lobby arrow color.
     * `$secid <section-id>`: Sets your override section ID. After running this command, any games you create will use your override section ID for rare drops instead of your character's actual section ID. To revert to your actual section id, run `$secid` with no name after it.
 
-* Blue Burst player commands
+* Blue Burst player commands (game server only)
     * `$bbchar <username> <password> <1-4>`: Use this command when playing on a non-BB version of PSO. If the username and password are correct, this command converts your current character to BB format and saves it on the server in the given slot.
     * `$edit <stat> <value>`: Modifies your character data.
     * `$item <data>`: Sets the next item to be dropped from an enemy or box.
 
-* Game state commands
+* Game state commands (game server only)
     * `$maxlevel <level>`: Sets the maximum level for players to join the current game.
     * `$minlevel <level>`: Sets the minimum level for players to join the current game.
     * `$password <password>`: Sets the game's join password. To unlock the game, run `$password` with nothing after it.
 
 * Cheat mode commands
-    * `$cheat`: Enables or disables cheat mode for the current game. All other cheat mode commands do nothing if cheat mode is disabled.
+    * `$cheat`: Enables or disables cheat mode for the current game. All other cheat mode commands do nothing if cheat mode is disabled. This command does nothing on the proxy server, since cheat mode is always enabled there.
     * `$infhp` / `$inftp`: Enables or disables infinite HP or TP mode. Applies to only you. In infinite HP mode, one-hit KO attacks will still kill you.
     * `$warp <area-id>`: Warps yourself to the given area.
-    * `$next`: Warps yourself to the next area.
+    * `$next` (game server only): Warps yourself to the next area.
     * `$swa`: Enables or disables switch assist. When enabled, the server will attempt to automatically unlock two-player doors in solo games if you step on both switches sequentially.
 
 * Configuration commands
-    * `$event <event>` / `$allevent <event>`: Sets the current holiday event in the current lobby, or in all lobbies. Holiday events are documented in the "Using $event" item in the information menu.
-    * `$song <song-id>`: Plays a specific song in the current lobby (Episode 3 only).
+    * `$event <event>`: Sets the current holiday event in the current lobby. Holiday events are documented in the "Using $event" item in the information menu. If you're on the proxy server, only you will see the new event; other players will not.
+    * `$allevent <event>` (game server only): Sets the current holiday event in all lobbies.
+    * `$song <song-id>` (game server only, Episode 3 only): Plays a specific song in the current lobby.
 
-* Administration commands
+* Administration commands (game server only)
     * `$ann <message>`: Sends an announcement message. The message text is sent to all players in all games and lobbies.
     * `$ax <message>`: Sends a message to the server's terminal. This cannot be used to run server shell commands; it only prints text to stderr.
-    * `$silence <identifier>`: Silences a player (remove their ability to chat) or unsilences a player. The identifier may be the player's name or guild card number.
-    * `$kick <identifier>`: Disconnects a player. The identifier may be the player's name or guild card number.
-    * `$ban <identifier>`: Bans a player. The identifier may be the player's name or guild card number.
+    * `$silence <identifier>`: Silences a player (remove their ability to chat) or unsilences a player. The identifier may be the player's name or Guild Card number.
+    * `$kick <identifier>`: Disconnects a player. The identifier may be the player's name or Guild Card number.
+    * `$ban <identifier>`: Bans a player. The identifier may be the player's name or Guild Card number.
 
 ### Using newserv as a proxy
 
@@ -103,8 +122,7 @@ To use the proxy, add an entry to the ProxyDestinations dictionary in config.jso
 
 A few things to be aware of when using the proxy server:
 - On PC and GC, using the Change Ship or Change Block actions from the lobby counter will bring you back to newserv's main menu, not the remote server's ship select. You can go back to the server you were just on by choosing it from newserv's proxy server menu again.
-- The remote server will probably try to assign you a guild card number that doesn't match the one you have on newserv. The proxy server rewrites the commands on the fly to make it look like the remote server assigned you the same guild card number as you have on newserv, but if the remote server has some external integrations (e.g. forum or Discord bots), they will use the guild card number that the remote server believes it has assigned to you. The number assigned by the remote server is shown to you when you connect to the remote server.
-- The proxy server blocks chat commands that look like newserv commands by default, but you can change this with the `set-chat-safety off` shell command if needed.
+- The remote server will probably try to assign you a Guild Card number that doesn't match the one you have on newserv. The proxy server rewrites the commands on the fly to make it look like the remote server assigned you the same Guild Card number as you have on newserv, but if the remote server has some external integrations (e.g. forum or Discord bots), they will use the Guild Card number that the remote server believes it has assigned to you. The number assigned by the remote server is shown to you when you first connect to the remote server, and you can retrieve it in lobbies or during games with the $li command.
 - There are shell commands that affect clients on the proxy (run 'help' in the shell to see what they are). All proxy commands in the shell only work when there's exactly one client connected through the proxy, since there isn't (yet) a way to say via the shell which session you want to affect.
 
 ### Connecting local clients

src/Channel.cc

@@ -0,0 +1,393 @@
+#include "Channel.hh"
+
+#include <event2/buffer.h>
+#include <event2/bufferevent.h>
+#include <event2/event.h>
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <phosg/Network.hh>
+#include <phosg/Time.hh>
+
+#include "Version.hh"
+
+using namespace std;
+
+
+
+extern bool use_terminal_colors;
+
+
+
+static void flush_and_free_bufferevent(struct bufferevent* bev) {
+  bufferevent_flush(bev, EV_READ | EV_WRITE, BEV_FINISHED);
+  bufferevent_free(bev);
+}
+
+
+
+Channel::Channel(
+    GameVersion version,
+    on_command_received_t on_command_received,
+    on_error_t on_error,
+    void* context_obj,
+    const string& name,
+    TerminalFormat terminal_send_color,
+    TerminalFormat terminal_recv_color)
+  : bev(nullptr, flush_and_free_bufferevent),
+    version(version),
+    name(name),
+    terminal_send_color(terminal_send_color),
+    terminal_recv_color(terminal_recv_color),
+    on_command_received(on_command_received),
+    on_error(on_error),
+    context_obj(context_obj) {
+}
+
+Channel::Channel(
+    struct bufferevent* bev,
+    GameVersion version,
+    on_command_received_t on_command_received,
+    on_error_t on_error,
+    void* context_obj,
+    const string& name,
+    TerminalFormat terminal_send_color,
+    TerminalFormat terminal_recv_color)
+  : bev(nullptr, flush_and_free_bufferevent),
+    version(version),
+    name(name),
+    terminal_send_color(terminal_send_color),
+    terminal_recv_color(terminal_recv_color),
+    on_command_received(on_command_received),
+    on_error(on_error),
+    context_obj(context_obj) {
+  this->set_bufferevent(bev);
+}
+
+void Channel::replace_with(
+    Channel&& other,
+    on_command_received_t on_command_received,
+    on_error_t on_error,
+    void* context_obj,
+    const std::string& name) {
+  this->set_bufferevent(other.bev.release());
+  this->local_addr = other.local_addr;
+  this->remote_addr = other.remote_addr;
+  this->is_virtual_connection = other.is_virtual_connection;
+  this->version = other.version;
+  this->crypt_in = other.crypt_in;
+  this->crypt_out = other.crypt_out;
+  this->name = name;
+  this->terminal_send_color = other.terminal_send_color;
+  this->terminal_recv_color = other.terminal_recv_color;
+  this->on_command_received = on_command_received;
+  this->on_error = on_error;
+  this->context_obj = context_obj;
+  other.disconnect(); // Clears crypts, addrs, etc.
+}
+
+void Channel::set_bufferevent(struct bufferevent* bev) {
+  this->bev.reset(bev);
+
+  if (this->bev.get()) {
+    int fd = bufferevent_getfd(this->bev.get());
+    if (fd < 0) {
+      this->is_virtual_connection = true;
+      memset(&this->local_addr, 0, sizeof(this->local_addr));
+      memset(&this->remote_addr, 0, sizeof(this->remote_addr));
+    } else {
+      this->is_virtual_connection = false;
+      get_socket_addresses(fd, &this->local_addr, &this->remote_addr);
+    }
+
+    bufferevent_setcb(this->bev.get(),
+        &Channel::dispatch_on_input, nullptr,
+        &Channel::dispatch_on_error, this);
+    bufferevent_enable(this->bev.get(), EV_READ | EV_WRITE);
+
+  } else {
+    this->is_virtual_connection = false;
+    memset(&this->local_addr, 0, sizeof(this->local_addr));
+    memset(&this->remote_addr, 0, sizeof(this->remote_addr));
+  }
+}
+
+
+
+void Channel::disconnect() {
+  if (this->bev.get()) {
+    // If the output buffer is not empty, move the bufferevent into the draining
+    // pool instead of disconnecting it, to make sure all the data gets sent.
+    struct evbuffer* out_buffer = bufferevent_get_output(this->bev.get());
+    if (evbuffer_get_length(out_buffer) == 0) {
+      this->bev.reset(); // Destructor flushes and frees the bufferevent
+    } else {
+      // The callbacks will free it when all the data is sent or the client
+      // disconnects
+
+      auto on_output = +[](struct bufferevent* bev, void*) -> void {
+        flush_and_free_bufferevent(bev);
+      };
+
+      auto on_error = +[](struct bufferevent* bev, short events, void*) -> void {
+        if (events & BEV_EVENT_ERROR) {
+          int err = EVUTIL_SOCKET_ERROR();
+          log(WARNING, "Disconnecting channel caused error %d (%s)", err,
+              evutil_socket_error_to_string(err));
+        }
+        if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
+          bufferevent_flush(bev, EV_WRITE, BEV_FINISHED);
+          bufferevent_free(bev);
+        }
+      };
+
+      struct bufferevent* bev = this->bev.release();
+      bufferevent_setcb(bev, nullptr, on_output, on_error, bev);
+      bufferevent_disable(bev, EV_READ);
+    }
+  }
+
+  memset(&this->local_addr, 0, sizeof(this->local_addr));
+  memset(&this->remote_addr, 0, sizeof(this->remote_addr));
+  this->is_virtual_connection = false;
+  this->crypt_in.reset();
+  this->crypt_out.reset();
+}
+
+
+
+Channel::Message Channel::recv(bool print_contents) {
+  struct evbuffer* buf = bufferevent_get_input(this->bev.get());
+
+  size_t header_size = (this->version == GameVersion::BB) ? 8 : 4;
+  PSOCommandHeader header;
+  if (evbuffer_copyout(buf, &header, header_size)
+      < static_cast<ssize_t>(header_size)) {
+    throw out_of_range("no command available");
+  }
+
+  if (this->crypt_in.get()) {
+    this->crypt_in->decrypt(&header, header_size, false);
+  }
+
+  size_t command_logical_size = header.size(version);
+
+  // If encryption is enabled, BB pads commands to 8-byte boundaries, and this
+  // is not reflected in the size field. This logic does not occur if encryption
+  // is not yet enabled.
+  size_t command_physical_size = (this->crypt_in.get() && (version == GameVersion::BB))
+      ? ((command_logical_size + 7) & ~7) : command_logical_size;
+  if (evbuffer_get_length(buf) < command_physical_size) {
+    throw out_of_range("no command available");
+  }
+
+  // If we get here, then there is a full command in the buffer. Some encryption
+  // algorithms' advancement depends on the decrypted data, so we have to
+  // actually decrypt the header again (with advance=true) to keep them in a
+  // consistent state.
+
+  string header_data(header_size, '\0');
+  if (evbuffer_remove(buf, header_data.data(), header_data.size())
+      < static_cast<ssize_t>(header_data.size())) {
+    throw logic_error("enough bytes available, but could not remove them");
+  }
+  if (this->crypt_in.get()) {
+    this->crypt_in->decrypt(header_data.data(), header_data.size());
+  }
+
+  string command_data(command_physical_size - header_size, '\0');
+  if (evbuffer_remove(buf, command_data.data(), command_data.size())
+      < static_cast<ssize_t>(command_data.size())) {
+    throw logic_error("enough bytes available, but could not remove them");
+  }
+  if (this->crypt_in.get()) {
+    this->crypt_in->decrypt(command_data.data(), command_data.size());
+  }
+  command_data.resize(command_logical_size - header_size);
+
+  if (print_contents && (this->terminal_recv_color != TerminalFormat::END)) {
+    if (use_terminal_colors && this->terminal_recv_color != TerminalFormat::NORMAL) {
+      print_color_escape(stderr, this->terminal_recv_color, TerminalFormat::BOLD, TerminalFormat::END);
+    }
+
+    string name_token;
+    if (!this->name.empty()) {
+      name_token = " from " + this->name;
+    }
+    log(INFO, "Received%s (version=%s command=%04hX flag=%08X)",
+        name_token.c_str(),
+        name_for_version(this->version),
+        header.command(this->version),
+        header.flag(this->version));
+
+    vector<struct iovec> iovs;
+    iovs.emplace_back(iovec{.iov_base = header_data.data(), .iov_len = header_data.size()});
+    iovs.emplace_back(iovec{.iov_base = command_data.data(), .iov_len = command_data.size()});
+    print_data(stderr, iovs, 0, nullptr, PrintDataFlags::PRINT_ASCII | PrintDataFlags::DISABLE_COLOR);
+
+    if (use_terminal_colors && this->terminal_recv_color != TerminalFormat::NORMAL) {
+      print_color_escape(stderr, TerminalFormat::NORMAL, TerminalFormat::END);
+    }
+  }
+
+  return {
+    .command = header.command(this->version),
+    .flag = header.flag(this->version),
+    .data = move(command_data),
+  };
+}
+
+void Channel::send(uint16_t cmd, uint32_t flag, const void* data, size_t size,
+    bool print_contents) {
+  if (!this->connected()) {
+    log(WARNING, "Attempted to send command on closed channel; dropping data");
+  }
+
+  string send_data;
+  size_t logical_size;
+  size_t send_data_size = 0;
+  switch (this->version) {
+    case GameVersion::GC:
+    case GameVersion::DC: {
+      PSOCommandHeaderDCGC header;
+      if (this->crypt_out.get()) {
+        send_data_size = (sizeof(header) + size + 3) & ~3;
+      } else {
+        send_data_size = (sizeof(header) + size);
+      }
+      logical_size = send_data_size;
+      header.command = cmd;
+      header.flag = flag;
+      header.size = send_data_size;
+      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
+      break;
+    }
+
+    case GameVersion::PC:
+    case GameVersion::PATCH: {
+      PSOCommandHeaderPC header;
+      if (this->crypt_out.get()) {
+        send_data_size = (sizeof(header) + size + 3) & ~3;
+      } else {
+        send_data_size = (sizeof(header) + size);
+      }
+      logical_size = send_data_size;
+      header.size = send_data_size;
+      header.command = cmd;
+      header.flag = flag;
+      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
+      break;
+    }
+
+    case GameVersion::BB: {
+      // BB has an annoying behavior here: command lengths must be multiples of
+      // 4, but the actual data length must be a multiple of 8. If the size
+      // field is not divisible by 8, 4 extra bytes are sent anyway. This
+      // behavior only applies when encryption is enabled - any commands sent
+      // before encryption is enabled have no size restrictions (except they
+      // must include a full header and must fit in the client's receive
+      // buffer), and no implicit extra bytes are sent.
+      PSOCommandHeaderBB header;
+      if (this->crypt_out.get()) {
+        send_data_size = (sizeof(header) + size + 7) & ~7;
+      } else {
+        send_data_size = (sizeof(header) + size);
+      }
+      logical_size = (sizeof(header) + size + 3) & ~3;
+      header.size = logical_size;
+      header.command = cmd;
+      header.flag = flag;
+      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
+      break;
+    }
+
+    default:
+      throw logic_error("unimplemented game version in send_command");
+  }
+
+  // All versions of PSO I've seen (PC, GC, BB) have a receive buffer 0x7C00
+  // bytes in size
+  if (send_data_size > 0x7C00) {
+    throw runtime_error("outbound command too large");
+  }
+
+  if (send_data.size() < send_data_size) {
+    send_data.append(reinterpret_cast<const char*>(data), size);
+    send_data.resize(send_data_size, '\0');
+  }
+
+  if (print_contents && (this->terminal_send_color != TerminalFormat::END)) {
+    string name_token;
+    if (!this->name.empty()) {
+      name_token = " to " + this->name;
+    }
+    if (use_terminal_colors && this->terminal_send_color != TerminalFormat::NORMAL) {
+      print_color_escape(stderr, TerminalFormat::FG_YELLOW, TerminalFormat::BOLD, TerminalFormat::END);
+    }
+    log(INFO, "Sending%s (version=%s command=%04hX flag=%08X)",
+        name_token.c_str(), name_for_version(version), cmd, flag);
+    print_data(stderr, send_data.data(), logical_size, 0, nullptr, PrintDataFlags::PRINT_ASCII | PrintDataFlags::DISABLE_COLOR);
+    if (use_terminal_colors && this->terminal_send_color != TerminalFormat::NORMAL) {
+      print_color_escape(stderr, TerminalFormat::NORMAL, TerminalFormat::END);
+    }
+  }
+
+  if (this->crypt_out.get()) {
+    this->crypt_out->encrypt(send_data.data(), send_data.size());
+  }
+
+  struct evbuffer* buf = bufferevent_get_output(this->bev.get());
+  evbuffer_add(buf, send_data.data(), send_data.size());
+}
+
+void Channel::send(uint16_t cmd, uint32_t flag, const string& data, bool print_contents) {
+  this->send(cmd, flag, data.data(), data.size(), print_contents);
+}
+
+void Channel::send(const void* data, size_t size, bool print_contents) {
+  size_t header_size = (this->version == GameVersion::BB) ? 8 : 4;
+  const auto* header = reinterpret_cast<const PSOCommandHeader*>(data);
+  this->send(
+      header->command(this->version),
+      header->flag(this->version),
+      reinterpret_cast<const uint8_t*>(data) + header_size,
+      size - header_size,
+      print_contents);
+}
+
+void Channel::send(const string& data, bool print_contents) {
+  return this->send(data.data(), data.size(), print_contents);
+}
+
+
+
+void Channel::dispatch_on_input(struct bufferevent*, void* ctx) {
+  Channel* ch = reinterpret_cast<Channel*>(ctx);
+  // The client can be disconnected during on_command_received, so we have to
+  // make sure ch->bev is valid every time before calling recv()
+  while (ch->bev.get()) {
+    Message msg;
+    try {
+      msg = ch->recv();
+    } catch (const out_of_range&) {
+      break;
+    } catch (const exception& e) {
+      log(WARNING, "Error receiving on channel: %s", e.what());
+      ch->on_error(*ch, BEV_EVENT_ERROR);
+      break;
+    }
+    if (ch->on_command_received) {
+      ch->on_command_received(*ch, msg.command, msg.flag, msg.data);
+    }
+  }
+}
+
+void Channel::dispatch_on_error(struct bufferevent*, short events, void* ctx) {
+  Channel* ch = reinterpret_cast<Channel*>(ctx);
+  if (ch->on_error) {
+    ch->on_error(*ch, events);
+  } else {
+    ch->disconnect();
+  }
+}

src/Channel.hh

@@ -0,0 +1,92 @@
+#pragma once
+
+#include <netinet/in.h>
+
+#include <memory>
+#include <string>
+
+#include "PSOEncryption.hh"
+#include "PSOProtocol.hh"
+#include "Version.hh"
+
+
+
+struct Channel {
+  std::unique_ptr<struct bufferevent, void (*)(struct bufferevent*)> bev;
+  struct sockaddr_storage local_addr;
+  struct sockaddr_storage remote_addr;
+  bool is_virtual_connection;
+
+  GameVersion version;
+  std::shared_ptr<PSOEncryption> crypt_in;
+  std::shared_ptr<PSOEncryption> crypt_out;
+
+  std::string name;
+  TerminalFormat terminal_send_color;
+  TerminalFormat terminal_recv_color;
+
+  struct Message {
+    uint16_t command;
+    uint32_t flag;
+    std::string data;
+  };
+
+  typedef void (*on_command_received_t)(Channel&, uint16_t, uint32_t, std::string&);
+  typedef void (*on_error_t)(Channel&, short);
+
+  on_command_received_t on_command_received;
+  on_error_t on_error;
+  void* context_obj;
+
+  Channel(
+      GameVersion version,
+      on_command_received_t on_command_received,
+      on_error_t on_error,
+      void* context_obj,
+      const std::string& name = "",
+      TerminalFormat terminal_send_color = TerminalFormat::END,
+      TerminalFormat terminal_recv_color = TerminalFormat::END);
+  Channel(
+      struct bufferevent* bev,
+      GameVersion version,
+      on_command_received_t on_command_received,
+      on_error_t on_error,
+      void* context_obj,
+      const std::string& name = "",
+      TerminalFormat terminal_send_color = TerminalFormat::END,
+      TerminalFormat terminal_recv_color = TerminalFormat::END);
+  Channel(const Channel& other) = delete;
+  Channel(Channel&& other) = delete;
+  Channel& operator=(const Channel& other) = delete;
+  Channel& operator=(Channel&& other) = delete;
+
+  void replace_with(
+      Channel&& other,
+      on_command_received_t on_command_received,
+      on_error_t on_error,
+      void* context_obj,
+      const std::string& name = "");
+
+  void set_bufferevent(struct bufferevent* bev);
+
+  inline bool connected() const {
+    return this->bev.get() != nullptr;
+  }
+  void disconnect();
+
+  // Receives a message. Throws std::out_of_range if no messages are available.
+  Message recv(bool print_contents = true);
+
+  // Sends a message with an automatically-constructed header.
+  void send(uint16_t cmd, uint32_t flag = 0, const void* data = nullptr, size_t size = 0, bool print_contents = true);
+  void send(uint16_t cmd, uint32_t flag, const std::string& data, bool print_contents = true);
+
+  // Sends a message with a pre-existing header (as the first few bytes in the
+  // data)
+  void send(const void* data = nullptr, size_t size = 0, bool print_contents = true);
+  void send(const std::string& data, bool print_contents = true);
+
+private:
+  static void dispatch_on_input(struct bufferevent*, void* ctx);
+  static void dispatch_on_error(struct bufferevent*, short events, void* ctx);
+};

src/ChatCommands.cc

@@ -9,6 +9,7 @@
 #include <phosg/Time.hh>
 
 #include "Server.hh"
+#include "ProxyServer.hh"
 #include "Lobby.hh"
 #include "Client.hh"
 #include "SendCommands.hh"
@@ -91,7 +92,7 @@ static void check_is_leader(shared_ptr<Lobby> l, shared_ptr<Client> c) {
 ////////////////////////////////////////////////////////////////////////////////
 // Message commands
 
-static void command_lobby_info(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_lobby_info(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string&) {
   // no preconditions - everyone can use this command
 
@@ -120,20 +121,61 @@ static void command_lobby_info(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   }
 }
 
-static void command_ax(shared_ptr<ServerState>, shared_ptr<Lobby>,
+static void proxy_command_lobby_info(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string&) {
+  vector<const char*> cheats_tokens;
+  if (session.switch_assist) {
+    cheats_tokens.emplace_back("SWA");
+  }
+  if (session.infinite_hp) {
+    cheats_tokens.emplace_back("HP");
+  }
+  if (session.infinite_tp) {
+    cheats_tokens.emplace_back("TP");
+  }
+  string cheats_str = cheats_tokens.empty() ? "none" : join(cheats_tokens, ",");
+
+  vector<const char*> behaviors_tokens;
+  if (session.save_files) {
+    behaviors_tokens.emplace_back("SF");
+  }
+  if (session.function_call_return_value >= 0) {
+    behaviors_tokens.emplace_back("BFC");
+  }
+  string behaviors_str = behaviors_tokens.empty() ? "none" : join(behaviors_tokens, ",");
+
+  string section_id_override = "none";
+  if (session.override_section_id >= 0) {
+    section_id_override = name_for_section_id(session.override_section_id);
+  }
+
+  send_text_message_printf(session.client_channel,
+      "$C7GC: $C6%" PRIu32 "\n"
+      "$C7Client ID: $C6%zu\n"
+      "$C7Cheats: $C6%s\n"
+      "$C7Flags: $C6%s\n"
+      "$C7SecID override: $C6%s\n",
+      session.remote_guild_card_number,
+      session.lobby_client_id,
+      cheats_str.c_str(),
+      behaviors_str.c_str(),
+      section_id_override.c_str());
+}
+
+static void server_command_ax(shared_ptr<ServerState>, shared_ptr<Lobby>,
     shared_ptr<Client> c, const std::u16string& args) {
   check_privileges(c, Privilege::ANNOUNCE);
   string message = encode_sjis(args);
   log(INFO, "[Client message from %010u] %s\n", c->license->serial_number, message.c_str());
 }
 
-static void command_announce(shared_ptr<ServerState> s, shared_ptr<Lobby>,
+static void server_command_announce(shared_ptr<ServerState> s, shared_ptr<Lobby>,
     shared_ptr<Client> c, const std::u16string& args) {
   check_privileges(c, Privilege::ANNOUNCE);
   send_text_message(s, args);
 }
 
-static void command_arrow(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_arrow(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   // no preconditions
   c->lobby_arrow_color = stoull(encode_sjis(args), nullptr, 0);
@@ -142,10 +184,20 @@ static void command_arrow(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   }
 }
 
+static void proxy_command_arrow(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string& args) {
+  session.server_channel.send(0x89, stoull(encode_sjis(args), nullptr, 0));
+}
+
+static void server_command_dbgid(shared_ptr<ServerState>, shared_ptr<Lobby>,
+    shared_ptr<Client> c, const std::u16string&) {
+  c->prefer_high_lobby_client_id = !c->prefer_high_lobby_client_id;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Lobby commands
 
-static void command_cheat(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_cheat(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string&) {
   check_is_game(l, true);
   check_is_leader(l, c);
@@ -169,7 +221,7 @@ static void command_cheat(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   }
 }
 
-static void command_lobby_event(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_lobby_event(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, false);
   check_privileges(c, Privilege::CHANGE_EVENT);
@@ -184,7 +236,24 @@ static void command_lobby_event(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   send_change_event(l, l->event);
 }
 
-static void command_lobby_event_all(shared_ptr<ServerState> s, shared_ptr<Lobby>,
+static void proxy_command_lobby_event(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string& args) {
+  if (args.empty()) {
+    session.override_lobby_event = -1;
+  } else {
+    uint8_t new_event = event_for_name(args);
+    if (new_event == 0xFF) {
+      send_text_message(session.client_channel, u"$C6No such lobby event.");
+    } else {
+      session.override_lobby_event = new_event;
+      if (session.version == GameVersion::GC || session.version == GameVersion::BB) {
+        session.client_channel.send(0xDA, session.override_lobby_event);
+      }
+    }
+  }
+}
+
+static void server_command_lobby_event_all(shared_ptr<ServerState> s, shared_ptr<Lobby>,
     shared_ptr<Client> c, const std::u16string& args) {
   check_privileges(c, Privilege::CHANGE_EVENT);
 
@@ -204,7 +273,7 @@ static void command_lobby_event_all(shared_ptr<ServerState> s, shared_ptr<Lobby>
   }
 }
 
-static void command_lobby_type(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_lobby_type(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, false);
   check_privileges(c, Privilege::CHANGE_EVENT);
@@ -230,20 +299,41 @@ static void command_lobby_type(shared_ptr<ServerState>, shared_ptr<Lobby> l,
 ////////////////////////////////////////////////////////////////////////////////
 // Game commands
 
-static void command_secid(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_secid(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, false);
 
   if (!args[0]) {
     c->override_section_id = -1;
-    send_text_message(l, u"$C6Override section ID\nremoved");
+    send_text_message(c, u"$C6Override section ID\nremoved");
   } else {
-    c->override_section_id = section_id_for_name(args);
-    send_text_message(l, u"$C6Override section ID\nset");
+    uint8_t new_secid = section_id_for_name(args);
+    if (new_secid == 0xFF) {
+      send_text_message(c, u"$C6Invalid section ID");
+    } else {
+      c->override_section_id = new_secid;
+      send_text_message(c, u"$C6Override section ID\nset");
+    }
   }
 }
 
-static void command_password(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void proxy_command_secid(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string& args) {
+  if (!args[0]) {
+    session.override_section_id = -1;
+    send_text_message(session.client_channel, u"$C6Override section ID\nremoved");
+  } else {
+    uint8_t new_secid = section_id_for_name(args);
+    if (new_secid == 0xFF) {
+      send_text_message(session.client_channel, u"$C6Invalid section ID");
+    } else {
+      session.override_section_id = new_secid;
+      send_text_message(session.client_channel, u"$C6Override section ID\nset");
+    }
+  }
+}
+
+static void server_command_password(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, true);
   check_is_leader(l, c);
@@ -260,7 +350,7 @@ static void command_password(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   }
 }
 
-static void command_min_level(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_min_level(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, true);
   check_is_leader(l, c);
@@ -271,7 +361,7 @@ static void command_min_level(shared_ptr<ServerState>, shared_ptr<Lobby> l,
       l->min_level + 1);
 }
 
-static void command_max_level(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_max_level(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, true);
   check_is_leader(l, c);
@@ -291,7 +381,7 @@ static void command_max_level(shared_ptr<ServerState>, shared_ptr<Lobby> l,
 ////////////////////////////////////////////////////////////////////////////////
 // Character commands
 
-static void command_edit(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
+static void server_command_edit(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, false);
   check_version(c, GameVersion::BB);
@@ -378,14 +468,15 @@ static void command_edit(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
 
 // TODO: implement this
 // TODO: make sure the bank name is filesystem-safe
-/* static void command_change_bank(shared_ptr<ServerState>, shared_ptr<Lobby>,
+/* static void server_command_change_bank(shared_ptr<ServerState>, shared_ptr<Lobby>,
     shared_ptr<Client> c, const std::u16string&) {
   check_version(c, GameVersion::BB);
 
   TODO
 } */
 
-static void command_convert_char_to_bb(shared_ptr<ServerState> s,
+// TODO: This can be implemented on the proxy server too.
+static void server_command_convert_char_to_bb(shared_ptr<ServerState> s,
     shared_ptr<Lobby> l, shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, false);
   check_not_version(c, GameVersion::BB);
@@ -433,7 +524,7 @@ static string name_for_client(shared_ptr<Client> c) {
   return "Player";
 }
 
-static void command_silence(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
+static void server_command_silence(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_privileges(c, Privilege::SILENCE_USER);
 
@@ -455,7 +546,7 @@ static void command_silence(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
       target->can_chat ? "un" : "");
 }
 
-static void command_kick(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
+static void server_command_kick(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_privileges(c, Privilege::KICK_USER);
 
@@ -477,7 +568,7 @@ static void command_kick(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
   send_text_message_printf(l, "$C6%s kicked off", target_name.c_str());
 }
 
-static void command_ban(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
+static void server_command_ban(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_privileges(c, Privilege::BAN_USER);
 
@@ -531,7 +622,7 @@ static void command_ban(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
 ////////////////////////////////////////////////////////////////////////////////
 // Cheat commands
 
-static void command_warp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_warp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, true);
   check_cheats_enabled(l);
@@ -560,7 +651,13 @@ static void command_warp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   send_warp(c, area);
 }
 
-static void command_next(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void proxy_command_warp(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string& args) {
+  uint32_t area = stoul(encode_sjis(args), nullptr, 0);
+  send_warp(session.client_channel, session.lobby_client_id, area);
+}
+
+static void server_command_next(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string&) {
   check_is_game(l, true);
   check_cheats_enabled(l);
@@ -579,38 +676,41 @@ static void command_next(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   send_warp(c, new_area);
 }
 
-static void command_what(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_what(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string&) {
   check_is_game(l, true);
   if (!l->episode || (l->episode > 3)) {
     return;
   }
-
-  float min_dist2 = 0.0f;
-  uint32_t nearest_item_id = 0xFFFFFFFF;
-  for (const auto& it : l->item_id_to_floor_item) {
-    if (it.second.area != c->area) {
-      continue;
-    }
-    float dx = it.second.x - c->x;
-    float dz = it.second.z - c->z;
-    float dist2 = (dx * dx) + (dz * dz);
-    if ((nearest_item_id == 0xFFFFFFFF) || (dist2 < min_dist2)) {
-      nearest_item_id = it.first;
-      min_dist2 = dist2;
-    }
-  }
-
-  if (nearest_item_id == 0xFFFFFFFF) {
-    send_text_message(c, u"No items are near you");
+  if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+    send_text_message(c, u"$C4Item tracking is off");
   } else {
-    const auto& item = l->item_id_to_floor_item.at(nearest_item_id);
-    string name = name_for_item(item.inv_item.data, true);
-    send_text_message(c, decode_sjis(name));
+    float min_dist2 = 0.0f;
+    uint32_t nearest_item_id = 0xFFFFFFFF;
+    for (const auto& it : l->item_id_to_floor_item) {
+      if (it.second.area != c->area) {
+        continue;
+      }
+      float dx = it.second.x - c->x;
+      float dz = it.second.z - c->z;
+      float dist2 = (dx * dx) + (dz * dz);
+      if ((nearest_item_id == 0xFFFFFFFF) || (dist2 < min_dist2)) {
+        nearest_item_id = it.first;
+        min_dist2 = dist2;
+      }
+    }
+
+    if (nearest_item_id == 0xFFFFFFFF) {
+      send_text_message(c, u"$C4No items are near you");
+    } else {
+      const auto& item = l->item_id_to_floor_item.at(nearest_item_id);
+      string name = name_for_item(item.inv_item.data, true);
+      send_text_message(c, decode_sjis(name));
+    }
   }
 }
 
-static void command_song(shared_ptr<ServerState>, shared_ptr<Lobby>,
+static void server_command_song(shared_ptr<ServerState>, shared_ptr<Lobby>,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_ep3(c, true);
 
@@ -618,7 +718,7 @@ static void command_song(shared_ptr<ServerState>, shared_ptr<Lobby>,
   send_ep3_change_music(c, song);
 }
 
-static void command_infinite_hp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void server_command_infinite_hp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string&) {
   check_is_game(l, true);
   check_cheats_enabled(l);
@@ -627,7 +727,14 @@ static void command_infinite_hp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   send_text_message_printf(c, "$C6Infinite HP %s", c->infinite_hp ? "enabled" : "disabled");
 }
 
-static void command_infinite_tp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void proxy_command_infinite_hp(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string&) {
+  session.infinite_hp = !session.infinite_hp;
+  send_text_message_printf(session.client_channel, "$C6Infinite HP %s",
+      session.infinite_hp ? "enabled" : "disabled");
+}
+
+static void server_command_infinite_tp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string&) {
   check_is_game(l, true);
   check_cheats_enabled(l);
@@ -636,7 +743,14 @@ static void command_infinite_tp(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   send_text_message_printf(c, "$C6Infinite TP %s", c->infinite_tp ? "enabled" : "disabled");
 }
 
-static void command_switch_assist(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void proxy_command_infinite_tp(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string&) {
+  session.infinite_tp = !session.infinite_tp;
+  send_text_message_printf(session.client_channel, "$C6Infinite TP %s",
+      session.infinite_tp ? "enabled" : "disabled");
+}
+
+static void server_command_switch_assist(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string&) {
   check_is_game(l, true);
   check_cheats_enabled(l);
@@ -645,7 +759,13 @@ static void command_switch_assist(shared_ptr<ServerState>, shared_ptr<Lobby> l,
   send_text_message_printf(c, "$C6Switch assist %s", c->switch_assist ? "enabled" : "disabled");
 }
 
-static void command_item(shared_ptr<ServerState>, shared_ptr<Lobby> l,
+static void proxy_command_switch_assist(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string&) {
+  session.switch_assist = !session.switch_assist;
+  send_text_message_printf(session.client_channel, "$C6Switch assist %s", session.switch_assist ? "enabled" : "disabled");
+}
+
+static void server_command_item(shared_ptr<ServerState>, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args) {
   check_is_game(l, true);
   check_cheats_enabled(l);
@@ -675,73 +795,113 @@ static void command_item(shared_ptr<ServerState>, shared_ptr<Lobby> l,
 
 ////////////////////////////////////////////////////////////////////////////////
 
-typedef void (*handler_t)(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
+typedef void (*server_handler_t)(shared_ptr<ServerState> s, shared_ptr<Lobby> l,
     shared_ptr<Client> c, const std::u16string& args);
+typedef void (*proxy_handler_t)(shared_ptr<ServerState>,
+    ProxyServer::LinkedSession& session, const std::u16string& args);
 struct ChatCommandDefinition {
-  handler_t handler;
+  server_handler_t server_handler;
+  proxy_handler_t proxy_handler;
   u16string usage;
 };
 
 static const unordered_map<u16string, ChatCommandDefinition> chat_commands({
     // TODO: implement command_help and actually use the usage strings here
-    {u"$allevent"  , {command_lobby_event_all   , u"Usage:\nallevent <name/ID>"}},
-    {u"$ann"       , {command_announce          , u"Usage:\nann <message>"}},
-    {u"$arrow"     , {command_arrow             , u"Usage:\narrow <color>"}},
-    {u"$ax"        , {command_ax                , u"Usage:\nax <message>"}},
-    {u"$ban"       , {command_ban               , u"Usage:\nban <name-or-number>"}},
-    {u"$bbchar"    , {command_convert_char_to_bb, u"Usage:\nbbchar <user> <pass> <1-4>"}},
-    // {u"$bank",       {command_bank              , u"Usage:\nbank <bank name>"}},
-    {u"$cheat"     , {command_cheat             , u"Usage:\ncheat"}},
-    {u"$edit"      , {command_edit              , u"Usage:\nedit <stat> <value>"}},
-    {u"$event"     , {command_lobby_event       , u"Usage:\nevent <name>"}},
-    {u"$infhp"     , {command_infinite_hp       , u"Usage:\ninfhp"}},
-    {u"$inftp"     , {command_infinite_tp       , u"Usage:\ninftp"}},
-    {u"$item"      , {command_item              , u"Usage:\nitem <item-code>"}},
-    {u"$kick"      , {command_kick              , u"Usage:\nkick <name-or-number>"}},
-    {u"$li"        , {command_lobby_info        , u"Usage:\nli"}},
-    {u"$maxlevel"  , {command_max_level         , u"Usage:\nmax_level <level>"}},
-    {u"$minlevel"  , {command_min_level         , u"Usage:\nmin_level <level>"}},
-    {u"$next"      , {command_next              , u"Usage:\nnext"}},
-    {u"$password"  , {command_password          , u"Usage:\nlock [password]\nomit password to\nunlock game"}},
-    {u"$secid"     , {command_secid             , u"Usage:\nsecid [section ID]\nomit section ID to\nrevert to normal"}},
-    {u"$silence"   , {command_silence           , u"Usage:\nsilence <name-or-number>"}},
-    {u"$song"      , {command_song              , u"Usage:\nsong <song-number>"}},
-    {u"$swa"       , {command_switch_assist     , u"Usage:\nswa"}},
-    {u"$type"      , {command_lobby_type        , u"Usage:\ntype <name>"}},
-    {u"$warp"      , {command_warp              , u"Usage:\nwarp <area-number>"}},
-    {u"$what"      , {command_what              , u"Usage:\nwhat"}},
+    {u"$allevent"  , {server_command_lobby_event_all   , nullptr                    , u"Usage:\nallevent <name/ID>"}},
+    {u"$ann"       , {server_command_announce          , nullptr                    , u"Usage:\nann <message>"}},
+    {u"$arrow"     , {server_command_arrow             , proxy_command_arrow        , u"Usage:\narrow <color>"}},
+    {u"$ax"        , {server_command_ax                , nullptr                    , u"Usage:\nax <message>"}},
+    {u"$ban"       , {server_command_ban               , nullptr                    , u"Usage:\nban <name-or-number>"}},
+    // TODO: implement this on proxy server
+    {u"$bbchar"    , {server_command_convert_char_to_bb, nullptr                    , u"Usage:\nbbchar <user> <pass> <1-4>"}},
+    {u"$cheat"     , {server_command_cheat             , nullptr                    , u"Usage:\ncheat"}},
+    {u"$dbgid"     , {server_command_dbgid             , nullptr                    , u"Usage:\ndbgid"}},
+    {u"$edit"      , {server_command_edit              , nullptr                    , u"Usage:\nedit <stat> <value>"}},
+    {u"$event"     , {server_command_lobby_event       , proxy_command_lobby_event  , u"Usage:\nevent <name>"}},
+    {u"$infhp"     , {server_command_infinite_hp       , proxy_command_infinite_hp  , u"Usage:\ninfhp"}},
+    {u"$inftp"     , {server_command_infinite_tp       , proxy_command_infinite_tp  , u"Usage:\ninftp"}},
+    {u"$item"      , {server_command_item              , nullptr                    , u"Usage:\nitem <item-code>"}},
+    {u"$kick"      , {server_command_kick              , nullptr                    , u"Usage:\nkick <name-or-number>"}},
+    {u"$li"        , {server_command_lobby_info        , proxy_command_lobby_info   , u"Usage:\nli"}},
+    {u"$maxlevel"  , {server_command_max_level         , nullptr                    , u"Usage:\nmax_level <level>"}},
+    {u"$minlevel"  , {server_command_min_level         , nullptr                    , u"Usage:\nmin_level <level>"}},
+    // TODO: implement this on proxy server
+    {u"$next"      , {server_command_next              , nullptr                    , u"Usage:\nnext"}},
+    {u"$password"  , {server_command_password          , nullptr                    , u"Usage:\nlock [password]\nomit password to\nunlock game"}},
+    {u"$secid"     , {server_command_secid             , proxy_command_secid        , u"Usage:\nsecid [section ID]\nomit section ID to\nrevert to normal"}},
+    {u"$silence"   , {server_command_silence           , nullptr                    , u"Usage:\nsilence <name-or-number>"}},
+    // TODO: implement this on proxy server
+    {u"$song"      , {server_command_song              , nullptr                    , u"Usage:\nsong <song-number>"}},
+    {u"$swa"       , {server_command_switch_assist     , proxy_command_switch_assist, u"Usage:\nswa"}},
+    {u"$type"      , {server_command_lobby_type        , nullptr                    , u"Usage:\ntype <name>"}},
+    {u"$warp"      , {server_command_warp              , proxy_command_warp         , u"Usage:\nwarp <area-number>"}},
+    {u"$what"      , {server_command_what              , nullptr                    , u"Usage:\nwhat"}},
 });
 
+struct SplitCommand {
+  u16string name;
+  u16string args;
+
+  SplitCommand(const u16string& text) {
+    size_t space_pos = text.find(u' ');
+    if (space_pos != string::npos) {
+      this->name = text.substr(0, space_pos);
+      this->args = text.substr(space_pos + 1);
+    } else {
+      this->name = text;
+    }
+  }
+};
+
 // This function is called every time any player sends a chat beginning with a
 // dollar sign. It is this function's responsibility to see if the chat is a
 // command, and to execute the command and block the chat if it is.
 void process_chat_command(std::shared_ptr<ServerState> s, std::shared_ptr<Lobby> l,
     std::shared_ptr<Client> c, const std::u16string& text) {
-
-  u16string command_name;
-  u16string text_str(text);
-  size_t space_pos = text_str.find(u' ');
-  if (space_pos != string::npos) {
-    command_name = text_str.substr(0, space_pos);
-    text_str = text_str.substr(space_pos + 1);
-  } else {
-    command_name = text_str;
-    text_str.clear();
-  }
+  SplitCommand cmd(text);
 
   const ChatCommandDefinition* def = nullptr;
   try {
-    def = &chat_commands.at(command_name);
+    def = &chat_commands.at(cmd.name);
   } catch (const out_of_range&) {
-    send_text_message(c, u"$C6Unknown command.");
+    send_text_message(c, u"$C6Unknown command");
     return;
   }
 
-  try {
-    def->handler(s, l, c, text_str.c_str());
-  } catch (const precondition_failed& e) {
-    send_text_message(c, e.what());
-  } catch (const exception& e) {
-    send_text_message_printf(c, "$C6Failed:\n%s", e.what());
+  if (!def->server_handler) {
+    send_text_message(c, u"$C6Command not available\non game server");
+  } else {
+    try {
+      def->server_handler(s, l, c, cmd.args);
+    } catch (const precondition_failed& e) {
+      send_text_message(c, e.what());
+    } catch (const exception& e) {
+      send_text_message_printf(c, "$C6Failed:\n%s", e.what());
+    }
+  }
+}
+
+void process_chat_command(std::shared_ptr<ServerState> s,
+    ProxyServer::LinkedSession& session, const std::u16string& text) {
+  SplitCommand cmd(text);
+
+  const ChatCommandDefinition* def = nullptr;
+  try {
+    def = &chat_commands.at(cmd.name);
+  } catch (const out_of_range&) {
+    send_text_message(session.client_channel, u"$C6Unknown command");
+    return;
+  }
+
+  if (!def->proxy_handler) {
+    send_text_message(session.client_channel, u"$C6Command not available\non proxy server");
+  } else {
+    try {
+      def->proxy_handler(s, session, cmd.args);
+    } catch (const precondition_failed& e) {
+      send_text_message(session.client_channel, e.what());
+    } catch (const exception& e) {
+      send_text_message_printf(session.client_channel, "$C6Failed:\n%s", e.what());
+    }
   }
 }

src/ChatCommands.hh

@@ -8,6 +8,9 @@
 #include "ServerState.hh"
 #include "Lobby.hh"
 #include "Client.hh"
+#include "ProxyServer.hh"
 
 void process_chat_command(std::shared_ptr<ServerState> s, std::shared_ptr<Lobby> l,
     std::shared_ptr<Client> c, const std::u16string& text);
+void process_chat_command(std::shared_ptr<ServerState> s,
+    ProxyServer::LinkedSession& session, const std::u16string& text);

src/Client.cc

@@ -24,37 +24,30 @@ Client::Client(
     GameVersion version,
     ServerBehavior server_behavior)
   : version(version),
+    bb_game_state(0),
     flags(flags_for_version(this->version, 0)),
-    bev(bev),
+    channel(bev, this->version, nullptr, nullptr, this, "", TerminalFormat::FG_YELLOW, TerminalFormat::FG_GREEN),
     server_behavior(server_behavior),
     should_disconnect(false),
+    should_send_to_lobby_server(false),
     proxy_destination_address(0),
     proxy_destination_port(0),
-    play_time_begin(now()),
-    last_recv_time(this->play_time_begin),
-    last_send_time(0),
     x(0.0f),
     z(0.0f),
     area(0),
     lobby_id(0),
     lobby_client_id(0),
     lobby_arrow_color(0),
+    prefer_high_lobby_client_id(false),
     next_exp_value(0),
     override_section_id(-1),
     infinite_hp(false),
     infinite_tp(false),
     switch_assist(false),
-    can_chat(true) {
+    can_chat(true),
+    pending_bb_save_player_index(0),
+    dol_base_addr(0) {
   this->last_switch_enabled_command.subcommand = 0;
-  int fd = bufferevent_getfd(this->bev);
-  if (fd < 0) {
-    this->is_virtual_connection = true;
-    memset(&this->local_addr, 0, sizeof(this->local_addr));
-    memset(&this->remote_addr, 0, sizeof(this->remote_addr));
-  } else {
-    this->is_virtual_connection = false;
-    get_socket_addresses(fd, &this->local_addr, &this->remote_addr);
-  }
   memset(&this->next_connection_addr, 0, sizeof(this->next_connection_addr));
 }
 
@@ -80,7 +73,7 @@ ClientConfigBB Client::export_config_bb() const {
   ClientConfigBB cc;
   cc.cfg = this->export_config();
   cc.bb_game_state = this->bb_game_state;
-  cc.bb_player_index = this->bb_player_index;
+  cc.bb_player_index = this->game_data.bb_player_index;
   cc.unused.clear(0xFF);
   return cc;
 }
@@ -97,5 +90,5 @@ void Client::import_config(const ClientConfig& cc) {
 void Client::import_config(const ClientConfigBB& cc) {
   this->import_config(cc.cfg);
   this->bb_game_state = cc.bb_game_state;
-  this->bb_player_index = cc.bb_player_index;
+  this->game_data.bb_player_index = cc.bb_player_index;
 }

src/Client.hh

@@ -4,13 +4,14 @@
 
 #include <memory>
 
+#include "Channel.hh"
+#include "CommandFormats.hh"
+#include "FunctionCompiler.hh"
 #include "License.hh"
 #include "Player.hh"
 #include "PSOEncryption.hh"
-
-#include "Text.hh"
 #include "PSOProtocol.hh"
-#include "CommandFormats.hh"
+#include "Text.hh"
 
 
 
@@ -25,11 +26,10 @@ struct Client {
     // After joining a lobby, client will no longer send D6 commands when they
     // close message boxes
     NO_MESSAGE_BOX_CLOSE_CONFIRMATION_AFTER_LOBBY_JOIN = 0x0002,
-    // Client has the above flag and has already joined a lobby, or is Blue Burst
-    // (BB never sends D6 commands)
+    // Client has the above flag and has already joined a lobby, or is not GC
     NO_MESSAGE_BOX_CLOSE_CONFIRMATION = 0x0004,
-    // Client is Episode 3, should be able to see CARD lobbies, and should only be
-    // able to see/join games with the IS_EPISODE_3 flag
+    // Client is Episode 3, should be able to see CARD lobbies, and should only
+    // be able to see/join games with the IS_EPISODE_3 flag
     EPISODE_3 = 0x0008,
     // Client is DC v1 (disables some features)
     DCV1 = 0x0010,
@@ -41,16 +41,20 @@ struct Client {
     IN_INFORMATION_MENU = 0x0080,
     // Client is at the welcome message (login server only)
     AT_WELCOME_MESSAGE = 0x0100,
+    // Client disconnect if it receives B2 (send_function_call)
+    DOES_NOT_SUPPORT_SEND_FUNCTION_CALL = 0x0200,
+    // Client has already received a 97 (enable saves) command, so don't show
+    // the programs menu anymore
+    SAVE_ENABLED = 0x0400,
 
-    // Note: There isn't a good way to detect Episode 3 until the player data is
-    // sent (via a 61 command), so the IS_EPISODE_3 flag is set in that handler
-    DEFAULT_V1 = DCV1,
-    DEFAULT_V2_DC = 0x0000,
-    DEFAULT_V2_PC = NO_MESSAGE_BOX_CLOSE_CONFIRMATION,
+    // TODO: Do DCv1 and PC support send_function_call? Here we assume they don't
+    DEFAULT_V1 = DCV1 | NO_MESSAGE_BOX_CLOSE_CONFIRMATION | DOES_NOT_SUPPORT_SEND_FUNCTION_CALL,
+    DEFAULT_V2_DC = NO_MESSAGE_BOX_CLOSE_CONFIRMATION,
+    DEFAULT_V2_PC = NO_MESSAGE_BOX_CLOSE_CONFIRMATION | DOES_NOT_SUPPORT_SEND_FUNCTION_CALL,
     DEFAULT_V3_GC = 0x0000,
-    DEFAULT_V3_GC_PLUS = NO_MESSAGE_BOX_CLOSE_CONFIRMATION_AFTER_LOBBY_JOIN,
-    DEFAULT_V3_GC_EP3 = NO_MESSAGE_BOX_CLOSE_CONFIRMATION_AFTER_LOBBY_JOIN | EPISODE_3,
-    DEFAULT_V4_BB = NO_MESSAGE_BOX_CLOSE_CONFIRMATION_AFTER_LOBBY_JOIN | NO_MESSAGE_BOX_CLOSE_CONFIRMATION,
+    DEFAULT_V3_GC_PLUS = NO_MESSAGE_BOX_CLOSE_CONFIRMATION_AFTER_LOBBY_JOIN | DOES_NOT_SUPPORT_SEND_FUNCTION_CALL,
+    DEFAULT_V3_GC_EP3 = NO_MESSAGE_BOX_CLOSE_CONFIRMATION_AFTER_LOBBY_JOIN | EPISODE_3 | DOES_NOT_SUPPORT_SEND_FUNCTION_CALL,
+    DEFAULT_V4_BB = NO_MESSAGE_BOX_CLOSE_CONFIRMATION_AFTER_LOBBY_JOIN | NO_MESSAGE_BOX_CLOSE_CONFIRMATION | SAVE_ENABLED,
   };
 
   // License & account
@@ -61,29 +65,17 @@ struct Client {
   // config can be up to 0x20 bytes; on BB it can be 0x28 bytes. We don't use
   // all of that space.
   uint8_t bb_game_state;
-  uint8_t bb_player_index;
   uint16_t flags;
 
-  // Encryption
-  std::shared_ptr<PSOEncryption> crypt_in;
-  std::shared_ptr<PSOEncryption> crypt_out;
-
   // Network
-  struct sockaddr_storage local_addr;
-  struct sockaddr_storage remote_addr;
-  struct bufferevent* bev;
+  Channel channel;
   struct sockaddr_storage next_connection_addr;
   ServerBehavior server_behavior;
-  bool is_virtual_connection;
   bool should_disconnect;
+  bool should_send_to_lobby_server;
   uint32_t proxy_destination_address;
   uint16_t proxy_destination_port;
 
-  // Timing & menus
-  uint64_t play_time_begin; // time of connection (used for incrementing play time on BB)
-  uint64_t last_recv_time; // time of last data received
-  uint64_t last_send_time; // time of last data sent
-
   // Lobby/positioning
   float x;
   float z;
@@ -91,6 +83,7 @@ struct Client {
   uint32_t lobby_id; // which lobby is this person in?
   uint8_t lobby_client_id; // which client number is this person?
   uint8_t lobby_arrow_color; // lobby arrow color ID
+  bool prefer_high_lobby_client_id;
   ClientGameData game_data;
 
   // Miscellaneous (used by chat commands)
@@ -104,8 +97,11 @@ struct Client {
   std::string pending_bb_save_username;
   uint8_t pending_bb_save_player_index;
 
-  Client(struct bufferevent* bev, GameVersion version,
-      ServerBehavior server_behavior);
+  // DOL file loading state
+  uint32_t dol_base_addr;
+  std::shared_ptr<DOLFileIndex::DOLFile> loading_dol_file;
+
+  Client(struct bufferevent* bev, GameVersion version, ServerBehavior server_behavior);
 
   void set_license(std::shared_ptr<const License> l);
 

src/Compression.cc

@@ -96,10 +96,11 @@ struct prs_compress_ctx {
   }
 };
 
-string prs_compress(const string& data) {
+string prs_compress(const void* vdata, size_t size) {
+  const uint8_t* data = reinterpret_cast<const uint8_t*>(vdata);
   prs_compress_ctx pc;
 
-  ssize_t data_ssize = static_cast<ssize_t>(data.size());
+  ssize_t data_ssize = static_cast<ssize_t>(size);
   ssize_t read_offset = 0;
   while (read_offset < data_ssize) {
 
@@ -117,8 +118,8 @@ string prs_compress(const string& data) {
       while ((this_size < 0x100) && // max copy size is 255 bytes
              ((this_offset + this_size) < 0) && // don't copy past the read offset
              (this_size <= data_ssize - read_offset) && // don't copy past the end
-             !memcmp(data.data() + read_offset + this_offset,
-                     data.data() + read_offset, this_size)) {
+             !memcmp(data + read_offset + this_offset, data + read_offset,
+                     this_size)) {
         this_size++;
       }
       this_size--;
@@ -143,6 +144,10 @@ string prs_compress(const string& data) {
   return pc.finish();
 }
 
+string prs_compress(const string& data) {
+  return prs_compress(data.data(), data.size());
+}
+
 
 
 static int16_t get_u8_or_eof(StringReader& r) {

src/Compression.hh

@@ -6,6 +6,8 @@
 
 
 
+std::string prs_compress(const void* vdata, size_t size);
 std::string prs_compress(const std::string& data);
+
 std::string prs_decompress(const std::string& data, size_t max_size = 0);
 size_t prs_decompress_size(const std::string& data, size_t max_size = 0);

src/DNSServer.cc

@@ -101,7 +101,7 @@ void DNSServer::on_receive_message(int fd, short) {
 
     } else if (bytes < 0x0C) {
       log(WARNING, "[DNSServer] input query too small");
-      print_data(stderr, input);
+      print_data(stderr, input.data(), bytes);
 
     } else {
       input.resize(bytes);

src/Episode3.cc

@@ -0,0 +1,894 @@
+#include "Episode3.hh"
+
+#include <stdint.h>
+
+#include <array>
+#include <phosg/Filesystem.hh>
+
+#include "Compression.hh"
+#include "Text.hh"
+
+using namespace std;
+
+
+
+static const vector<const char*> name_for_card_type({
+  "HunterSC",
+  "ArkzSC",
+  "Item",
+  "Creature",
+  "Action",
+  "Assist",
+});
+
+static const unordered_map<uint8_t, const char*> description_for_when({
+  {0x01, "Set"}, // TODO: Is 01 this, or "Permanent"?
+  {0x02, "Attack"},
+  {0x03, "??? (TODO)"},
+  {0x04, "Before turn"},
+  {0x05, "Destroyed"},
+  {0x0A, "Permanent"}, // only used on Tollaw; could be same as 01
+  {0x0B, "Battle"},
+  {0x0C, "Opponent destroyed"}, // TODO: but this is also used for some support things like Shifta, and for Snatch, which also applies when opponents are not destroyed
+  {0x0D, "Attack lands"},
+  {0x0E, "Before attack phase"},
+  {0x16, "Battle end"},
+  {0x17, "Each defense"},
+  {0x20, "Each attack"},
+  {0x22, "Act phase"},
+  {0x27, "Move phase"},
+  {0x29, "Set and act phases"},
+  {0x33, "Defense phase"},
+  {0x3D, "Battle"}, // TODO: how is this different from 3D and 0B?
+  {0x3E, "Battle"}, // TODO: how is this different from 3D and 0B?
+  {0x3F, "Each defense"}, // TODO: how is this different from 17?
+  {0x46, "On specific turn"},
+});
+
+static const unordered_map<string, const char*> description_for_expr_token({
+  {"f",   "Number of FCs controlled by current SC"},
+  {"d",   "Die roll"},
+  {"ap",  "Attacker AP"}, // Unused
+  {"tp",  "Attacker TP"},
+  {"hp",  "Attacker HP"}, // TODO: How is this different from ehp?
+  {"mhp", "Attacker maximum HP"},
+  {"dm",  "Unknown: dm"}, // Unused
+  {"tdm", "Unknown: tdm"}, // Unused
+  {"tf",  "Number of SC\'s destroyed FCs"},
+  {"ac",  "Remaining ATK points"},
+  {"php", "Unknown: php"}, // Unused
+  {"dc",  "Unknown: dc"}, // Unused
+  {"cs",  "Unknown: cs"}, // Unused
+  {"a",   "Unknown: a"}, // Unused
+  {"kap", "Action cards AP"},
+  {"ktp", "Action cards TP"},
+  {"dn",  "Unknown: dn"}, // Unused
+  {"hf",  "Unknown: hf"}, // Unused
+  {"df",  "Number of destroyed ally FCs (including SC\'s own)"},
+  {"ff",  "Number of ally FCs (including SC\'s own)"},
+  {"ef",  "Number of enemy FCs"},
+  {"bi",  "Number of Native FCs on either team"},
+  {"ab",  "Number of A.Beast FCs on either team"},
+  {"mc",  "Number of Machine FCs on either team"},
+  {"dk",  "Number of Dark FCs on either team"},
+  {"sa",  "Number of Sword-type items on either team"},
+  {"gn",  "Number of Gun-type items on either team"},
+  {"wd",  "Number of Cane-type items on either team"},
+  {"tt",  "Unknown: tt"}, // Unused
+  {"lv",  "Dice bonus"},
+  {"adm", "Attack damage"},
+  {"ddm", "Defending damage"},
+  {"sat", "Number of Sword-type items on SC\'s team"},
+  {"edm", "Defending damage"}, // TODO: How is this different from ddm?
+  {"ldm", "Unknown: ldm"}, // Unused
+  {"rdm", "Defending damage"}, // TODO: How is this different from ddm/edm?
+  {"fdm", "Final damage (after defense)"},
+  {"ndm", "Unknown: ndm"}, // Unused
+  {"ehp", "Attacker HP"},
+});
+
+// Arguments are encoded as 3-character null-terminated strings (why?!), and are
+// used for adding conditions to effects (e.g. making them only trigger in
+// certain situations) or otherwise customizing their results.
+// Argument meanings:
+// a01 = ???
+// cXY/CXY = linked items (require item with cYX/CYX to be equipped as well)
+// dXY = roll one die; require result between X and Y inclusive
+// e00 = effect lasts while equipped? (in contrast to tXX)
+// hXX = require HP >= XX
+// iXX = require HP <= XX
+// nXX = require condition XX (see description_for_n_condition)
+// oXX = seems to be "require previous random-condition effect to have happened"
+//       TODO: this is used as both o01 (recovery) and o11 (reflection)
+//             conditions - why the difference?
+// pXX = who to target (see description_for_p_target)
+// rXX = randomly pass with XX% chance (if XX == 00, 100% chance?)
+// sXY = require card cost between X and Y ATK points (inclusive)
+// tXX = lasts XX turns, or activate after XX turns
+
+static const vector<const char*> description_for_n_condition({
+  /* n00 */ "Always true",
+  /* n01 */ "??? (TODO)",
+  /* n02 */ "Destroyed with a single attack?",
+  /* n03 */ "Unknown", // Unused
+  /* n04 */ "Attack has Pierce",
+  /* n05 */ "Attack has Rampage",
+  /* n06 */ "Native attribute",
+  /* n07 */ "A.Beast attribute",
+  /* n08 */ "Machine attribute",
+  /* n09 */ "Dark attribute",
+  /* n10 */ "Sword-type item",
+  /* n11 */ "Gun-type item",
+  /* n12 */ "Cane-type item",
+  /* n13 */ "Guard item",
+  /* n14 */ "Story Character",
+  /* n15 */ "Attacker does not use action cards",
+  /* n16 */ "Aerial attribute",
+  /* n17 */ "Same AP as opponent",
+  /* n18 */ "Opponent is SC",
+  /* n19 */ "Has Paralyzed condition",
+  /* n20 */ "Has Frozen condition",
+});
+
+static const vector<const char*> description_for_p_target({
+  /* p00 */ "Unknown: p00", // Unused; probably invalid
+  /* p01 */ "SC / FC who set the card",
+  /* p02 */ "Attacking SC / FC",
+  /* p03 */ "Unknown: p03", // Unused
+  /* p04 */ "Unknown: p04", // Unused
+  /* p05 */ "Unknown: p05", // Unused
+  /* p06 */ "??? (TODO)",
+  /* p07 */ "??? (TODO; Weakness)",
+  /* p08 */ "FC's owner SC",
+  /* p09 */ "Unknown: p09", // Unused
+  /* p10 */ "All ally FCs",
+  /* p11 */ "All ally FCs", // TODO: how is this different from p10?
+  /* p12 */ "All non-aerial FCs on both teams",
+  /* p13 */ "All FCs on both teams that are Frozen",
+  /* p14 */ "All FCs on both teams that have <= 3 HP",
+  /* p15 */ "All FCs except SCs", // TODO: used during attacks only?
+  /* p16 */ "All FCs except SCs", // TODO: used during attacks only? how is this different from p15?
+  /* p17 */ "This card",
+  /* p18 */ "SC who equipped this card",
+  /* p19 */ "Unknown: p19", // Unused
+  /* p20 */ "Unknown: p20", // Unused
+  /* p21 */ "Unknown: p21", // Unused
+  /* p22 */ "All characters (SCs & FCs) including this card", // TODO: But why does Shifta apply only to allies then?
+  /* p23 */ "All characters (SCs & FCs) except this card",
+  /* p24 */ "All FCs on both teams that have Paralysis",
+  /* p25 */ "Unknown: p25", // Unused
+  /* p26 */ "Unknown: p26", // Unused
+  /* p27 */ "Unknown: p27", // Unused
+  /* p28 */ "Unknown: p28", // Unused
+  /* p29 */ "Unknown: p29", // Unused
+  /* p30 */ "Unknown: p30", // Unused
+  /* p31 */ "Unknown: p31", // Unused
+  /* p32 */ "Unknown: p32", // Unused
+  /* p33 */ "Unknown: p33", // Unused
+  /* p34 */ "Unknown: p34", // Unused
+  /* p35 */ "All characters (SCs & FCs) within range", // Used for Explosion effect
+  /* p36 */ "All ally SCs within range, but not the caster", // Resta
+  /* p37 */ "All FCs or all opponent FCs (TODO)", // TODO: when to use which selector? is a3 involved here somehow?
+  /* p38 */ "All allies except items within range (and not this card)",
+  /* p39 */ "All FCs that cost 4 or more points",
+  /* p40 */ "All FCs that cost 3 or fewer points",
+  /* p41 */ "Unknown: p41", // Unused
+  /* p42 */ "Attacker during defense phase", // Only used by TP Defense
+  /* p43 */ "Owner SC of defending FC during attack",
+  /* p44 */ "SC\'s own creature FCs within range",
+  /* p45 */ "Both attacker and defender", // Used for Snatch, which moves EXP from one to the other
+  /* p46 */ "All SCs & FCs one space left or right of this card",
+  /* p47 */ "FC\'s owner Boss SC", // Only used for Gibbles+ which explicitly mentions Boss SC, so it looks like this is p08 but for bosses
+  /* p48 */ "Everything within range, including this card\'s user", // Madness
+  /* p49 */ "All ally FCs within range except this card",
+});
+
+struct Ep3AbilityDescription {
+  uint8_t command;
+  bool has_expr;
+  const char* name;
+  const char* description;
+};
+
+static const std::vector<Ep3AbilityDescription> name_for_effect_command({
+  {0x00, false, nullptr, nullptr},
+  {0x01, true,  "AP Boost", "Temporarily increase AP by N"},
+  {0x02, false, "Rampage", "Rampage"},
+  {0x03, true,  "Multi Strike", "Duplicate attack N times"},
+  {0x04, true,  "Damage Modifier 1", "Set attack damage / AP to N after action cards applied (step 1)"},
+  {0x05, false, "Immobile", "Give Immobile condition"},
+  {0x06, false, "Hold", "Give Hold condition"},
+  {0x07, false, nullptr, nullptr},
+  {0x08, true,  "TP Boost", "Add N TP temporarily during attack"},
+  {0x09, true,  "Give Damage", "Cause direct N HP loss"},
+  {0x0A, false, "Guom", "Give Guom condition"},
+  {0x0B, false, "Paralyze", "Give Paralysis condition"},
+  {0x0C, false, nullptr, nullptr},
+  {0x0D, false, "A/H Swap", "Swap AP and HP temporarily"},
+  {0x0E, false, "Pierce", "Attack SC directly even if they have items equipped"},
+  {0x0F, false, nullptr, nullptr},
+  {0x10, true,  "Heal", "Increase HP by N"},
+  {0x11, false, "Return to Hand", "Return card to hand"},
+  {0x12, false, nullptr, nullptr},
+  {0x13, false, nullptr, nullptr},
+  {0x14, false, "Acid", "Give Acid condition"},
+  {0x15, false, nullptr, nullptr},
+  {0x16, true,  "Mighty Knuckle", "Temporarily increase AP by N, and set ATK dice to zero"},
+  {0x17, true,  "Unit Blow", "Temporarily increase AP by N * number of this card set within phase"},
+  {0x18, false, "Curse", "Give Curse condition"},
+  {0x19, false, "Combo (AP)", "Temporarily increase AP by number of this card set within phase"},
+  {0x1A, false, "Pierce/Rampage Block", "Block attack if Pierce/Rampage (?)"},
+  {0x1B, false, "Ability Trap", "Temporarily disable opponent abilities"},
+  {0x1C, false, "Freeze", "Give Freeze condition"},
+  {0x1D, false, "Anti-Abnormality", "Cure all conditions"},
+  {0x1E, false, nullptr, nullptr},
+  {0x1F, false, "Explosion", "Damage all SCs and FCs by number of this same card set * 2"},
+  {0x20, false, nullptr, nullptr},
+  {0x21, false, nullptr, nullptr},
+  {0x22, false, nullptr, nullptr},
+  {0x23, false, "Return to Deck", "Cancel discard and move to bottom of deck instead"},
+  {0x24, false, "Aerial", "Give Aerial status"},
+  {0x25, true,  "AP Loss", "Make attacker temporarily lose N AP during defense"},
+  {0x26, true,  "Bonus From Leader", "Gain AP equal to the number of cards of type N on the field"},
+  {0x27, false, "Free Maneuver", "Enable movement over occupied tiles"},
+  {0x28, false, "Haste", "Make move actions free"},
+  {0x29, true,  "Clone", "Make setting this card free if at least one card of type N is already on the field"},
+  {0x2A, true,  "DEF Disable by Cost", "Disable use of any defense cards costing between (N / 10) and (N % 10) points, inclusive"},
+  {0x2B, true,  "Filial", "Increase controlling SC\'s HP by N when this card is destroyed"},
+  {0x2C, true,  "Snatch", "Steal N EXP during attack"},
+  {0x2D, true,  "Hand Disrupter", "DIscard N cards from hand immediately"},
+  {0x2E, false, "Drop", "Give Drop condition"},
+  {0x2F, false, "Action Disrupter", "Destroy all action cards used by attacker"},
+  {0x30, true,  "Set HP", "Set HP to N (?) (TODO)"},
+  {0x31, false, "Native Shield", "Block attacks from Native creatures"},
+  {0x32, false, "A.Beast Shield", "Block attacks from A.Beast creatures"},
+  {0x33, false, "Machine Shield", "Block attacks from Machine creatures"},
+  {0x34, false, "Dark Shield", "Block attacks from Dark creatures"},
+  {0x35, false, "Sword Shield", "Block attacks from Sword items"},
+  {0x36, false, "Gun Shield", "Block attacks from Gun items"},
+  {0x37, false, "Cane Shield", "Block attacks from Cane items"},
+  {0x38, false, nullptr, nullptr},
+  {0x39, false, nullptr, nullptr},
+  {0x3A, false, "Defender", "Make attacks go to setter of this card instead of original target"},
+  {0x3B, false, "Survival Decoys", "Redirect damage for multi-sided attack"},
+  {0x3C, true,  "Give/Take EXP", "Give N EXP, or take if N is negative"},
+  {0x3D, false, nullptr, nullptr},
+  {0x3E, false, "Death Companion", "If this card has 1 or 2 HP, set its HP to N"},
+  {0x3F, true,  "EXP Decoy", "If defender has EXP, lose EXP instead of getting damage when attacked"},
+  {0x40, true,  "Set MV", "Set MV to N"},
+  {0x41, true,  "Group", "Temporarily increase AP by N * number of this card on field, excluding itself"},
+  {0x42, false, "Berserk", "User of this card receives the same damage as target, and isn't helped by target's defense cards"},
+  {0x43, false, "Guard Creature", "Attacks on controlling SC damage this card instead"},
+  {0x44, false, "Tech", "Technique cards cost 1 fewer ATK point"},
+  {0x45, false, "Big Swing", "Increase all attacking ATK costs by 1"},
+  {0x46, false, nullptr, nullptr},
+  {0x47, false, "Shield Weapon", "Limit attacker\'s choice of target to guard items"},
+  {0x48, false, "ATK Dice Boost", "Increase ATK dice roll by 1"},
+  {0x49, false, nullptr, nullptr},
+  {0x4A, false, "Major Pierce", "If SC has over half of max HP, attacks target SC instead of equipped items"},
+  {0x4B, false, "Heavy Pierce", "If SC has 3 or more items equipped, attacks target SC instead of equipped items"},
+  {0x4C, false, "Major Rampage", "If SC has over half of max HP, attacks target SC and all equipped items"},
+  {0x4D, false, "Heavy Rampage", "If SC has 3 or more items equipped, attacks target SC and all equipped items"},
+  {0x4E, true,  "AP Growth", "Permanently increase AP by N"},
+  {0x4F, true,  "TP Growth", "Permanently increase TP by N"},
+  {0x50, true,  "Reborn", "If any card of type N is on the field, this card goes to the hand when destroyed instead of being discarded"},
+  {0x51, true,  "Copy", "Temporarily set AP/TP to N percent (or 100% if N is 0) of opponent\'s values"},
+  {0x52, false, nullptr, nullptr},
+  {0x53, true,  "Misc. Guards", "Add N to card's defense value"},
+  {0x54, true,  "AP Override", "Set AP to N temporarily"},
+  {0x55, true,  "TP Override", "Set TP to N temporarily"},
+  {0x56, false, "Return", "Return card to hand on destruction instead of discarding"},
+  {0x57, false, "A/T Swap Perm", "Permanently swap AP and TP"},
+  {0x58, false, "A/H Swap Perm", "Permanently swap AP and HP"},
+  {0x59, true,  "Slayers/Assassins", "Temporarily increase AP during attack"},
+  {0x5A, false, "Anti-Abnormality", "Remove all conditions"},
+  {0x5B, false, "Fixed Range", "Use SC\'s range instead of weapon or attack card ranges"},
+  {0x5C, false, "Elude", "SC does not lose HP when equipped items are destroyed"},
+  {0x5D, false, "Parry", "Forward attack to a random FC within one tile of original target, excluding attacker and original target"},
+  {0x5E, false, "Block Attack", "Completely block attack"},
+  {0x5F, false, nullptr, nullptr},
+  {0x60, false, nullptr, nullptr},
+  {0x61, true,  "Combo (TP)", "Gain TP equal to the number of cards of type N on the field"},
+  {0x62, true,  "Misc. AP Bonuses", "Temporarily increase AP by N"},
+  {0x63, true,  "Misc. TP Bonuses", "Temporarily increase TP by N"},
+  {0x64, false, nullptr, nullptr},
+  {0x65, true,  "Misc. Defense Bonuses", "Decrease damage by N"},
+  {0x66, true,  "Mostly Halfguards", "Reduce damage from incoming attack by N"},
+  {0x67, false, "Periodic Field", "Swap immunity to tech or physical attacks"},
+  {0x68, false, "Unlimited Summoning", "Allow unlimited summoning"},
+  {0x69, false, nullptr, nullptr},
+  {0x6A, true,  "MV Bonus", "Increase MV by N"},
+  {0x6B, true,  "Forward Damage", "Give N damage back to attacker during defense (?) (TODO)"},
+  {0x6C, true,  "Weak Spot / Influence", "Temporarily decrease AP by N"},
+  {0x6D, true,  "Damage Modifier 2", "Set attack damage / AP after action cards applied (step 2)"},
+  {0x6E, true,  "Weak Hit Block", "Block all attacks of N damage or less"},
+  {0x6F, true,  "AP Silence", "Temporarily decrease AP of opponent by N"},
+  {0x70, true,  "TP Silence", "Temporarily decrease TP of opponent by N"},
+  {0x71, false, "A/T Swap", "Temporarily swap AP and TP"},
+  {0x72, true,  "Halfguard", "Halve damage from attacks that would inflict N or more damage"},
+  {0x73, false, nullptr, nullptr},
+  {0x74, true,  "Rampage AP Loss", "Temporarily reduce AP by N"},
+  {0x75, false, nullptr, nullptr},
+  {0x76, false, "Reflect", "Generate reverse attack"},
+});
+
+void Ep3CardStats::Stat::decode_code() {
+  this->type = static_cast<Type>(this->code / 1000);
+  int16_t value = this->code - (this->type * 1000);
+  if (value != 999) {
+    switch (this->type) {
+      case Type::BLANK:
+        this->stat = 0;
+        break;
+      case Type::STAT:
+      case Type::PLUS_STAT:
+      case Type::EQUALS_STAT:
+        this->stat = value;
+        break;
+      case Type::MINUS_STAT:
+        this->stat = -value;
+        break;
+      default:
+        throw runtime_error("invalid card stat type");
+    }
+  } else {
+    this->stat = 0;
+    this->type = static_cast<Type>(this->type + 4);
+  }
+}
+
+string Ep3CardStats::Stat::str() const {
+  switch (this->type) {
+    case Type::BLANK:
+      return "";
+    case Type::STAT:
+      return string_printf("%hhd", this->stat);
+    case Type::PLUS_STAT:
+      return string_printf("+%hhd", this->stat);
+    case Type::MINUS_STAT:
+      return string_printf("-%d", -this->stat);
+    case Type::EQUALS_STAT:
+      return string_printf("=%hhd", this->stat);
+    case Type::UNKNOWN:
+      return "?";
+    case Type::PLUS_UNKNOWN:
+      return "+?";
+    case Type::MINUS_UNKNOWN:
+      return "-?";
+    case Type::EQUALS_UNKNOWN:
+      return "=?";
+    default:
+      return string_printf("[%02hhX %02hhX]", this->type, this->stat);
+  }
+}
+
+
+
+bool Ep3CardStats::Effect::is_empty() const {
+  return (this->command == 0 &&
+          this->expr.is_filled_with(0) &&
+          this->when == 0 &&
+          this->arg1.is_filled_with(0) &&
+          this->arg2.is_filled_with(0) &&
+          this->arg3.is_filled_with(0) &&
+          this->unknown_a3.is_filled_with(0));
+}
+
+string Ep3CardStats::Effect::str_for_arg(const std::string& arg) {
+  if (arg.empty()) {
+    return arg;
+  }
+  if (arg.size() != 3) {
+    return arg + "/(invalid)";
+  }
+  size_t value;
+  try {
+    value = stoul(arg.c_str() + 1, nullptr, 10);
+  } catch (const invalid_argument&) {
+    return arg + "/(invalid)";
+  }
+
+  switch (arg[0]) {
+    case 'a':
+      return arg + "/(unknown)";
+    case 'C':
+    case 'c':
+      return string_printf("%s/Req. linked item (%zu=>%zu)", arg.c_str(), value / 10, value % 10);
+    case 'd':
+      return string_printf("%s/Req. die roll in [%zu, %zu]", arg.c_str(), value / 10, value % 10);
+    case 'e':
+      return arg + "/While equipped";
+    case 'h':
+      return string_printf("%s/Req. HP >= %zu", arg.c_str(), value);
+    case 'i':
+      return string_printf("%s/Req. HP <= %zu", arg.c_str(), value);
+    case 'n':
+      try {
+        return string_printf("%s/Req. condition: %s", arg.c_str(), description_for_n_condition.at(value));
+      } catch (const out_of_range&) {
+        return arg + "/(unknown)";
+      }
+    case 'o':
+      return arg + "/Req. prev effect conditions passed";
+    case 'p':
+      try {
+        return string_printf("%s/Target: %s", arg.c_str(), description_for_p_target.at(value));
+      } catch (const out_of_range&) {
+        return arg + "/(unknown)";
+      }
+    case 'r':
+      return string_printf("%s/Req. random with %zu%% chance", arg.c_str(), value == 0 ? 100 : value);
+    case 's':
+      return string_printf("%s/Req. cost in [%zu, %zu]", arg.c_str(), value / 10, value % 10);
+    case 't':
+      return string_printf("%s/Turns: %zu", arg.c_str(), value);
+    default:
+      return arg + "/(unknown)";
+  }
+}
+
+string Ep3CardStats::Effect::str() const {
+  string cmd_str = string_printf("%02hhX", this->command);
+  try {
+    const char* name = name_for_effect_command.at(this->command).name;
+    if (name) {
+      cmd_str += ':';
+      cmd_str += name;
+    }
+  } catch (const out_of_range&) { }
+
+  string when_str = string_printf("%02hhX", this->when);
+  try {
+    const char* name = description_for_when.at(this->when);
+    if (name) {
+      when_str += ':';
+      when_str += name;
+    }
+  } catch (const out_of_range&) { }
+
+  string expr_str = this->expr;
+  if (!expr_str.empty()) {
+    expr_str = ", expr=" + expr_str;
+  }
+
+  string arg1str = this->str_for_arg(this->arg1);
+  string arg2str = this->str_for_arg(this->arg2);
+  string arg3str = this->str_for_arg(this->arg3);
+  string a3str = format_data_string(this->unknown_a3.data(), sizeof(this->unknown_a3));
+  return string_printf("(cmd=%s%s, when=%s, arg1=%s, arg2=%s, arg3=%s, a3=%s)",
+      cmd_str.c_str(), expr_str.c_str(), when_str.c_str(), arg1str.data(),
+      arg2str.data(), arg3str.data(), a3str.c_str());
+}
+
+
+
+void Ep3CardStats::decode_range() {
+  // If the cell representing the FC is nonzero, the card has a range from a
+  // list of constants. Otherwise, its range is already defined in the range
+  // array and should be left alone.
+  uint8_t index = (this->range[4] >> 8) & 0xF;
+  if (index != 0) {
+    this->range.clear(0);
+    switch (index) {
+      case 1: // Single cell in front of FC
+        this->range[3] = 0x00000100;
+        break;
+      case 2: // Cell in front of FC and the front-left and front-right (Slash)
+        this->range[3] = 0x00001110;
+        break;
+      case 3: // 3 cells in a line in front of FC
+        this->range[1] = 0x00000100;
+        this->range[2] = 0x00000100;
+        this->range[3] = 0x00000100;
+        break;
+      case 4: // All 8 cells around FC
+        this->range[3] = 0x00001110;
+        this->range[4] = 0x00001010;
+        this->range[5] = 0x00001110;
+        break;
+      case 5: // 2 cells in a line in front of FC
+        this->range[2] = 0x00000100;
+        this->range[3] = 0x00000100;
+        break;
+      case 6: // Entire field (renders as "A")
+        for (size_t x = 0; x < 6; x++) {
+          this->range[x] = 0x000FFFFF;
+        }
+        break;
+      case 7: // Superposition of 4 and 5 (unused)
+        this->range[2] = 0x00000100;
+        this->range[3] = 0x00001110;
+        this->range[4] = 0x00001010;
+        this->range[5] = 0x00001110;
+        break;
+      case 8: // All 8 cells around FC and FC's cell
+        this->range[3] = 0x00001110;
+        this->range[4] = 0x00001110;
+        this->range[5] = 0x00001110;
+        break;
+      case 9: // No cells
+        break;
+      // The table in the DOL file only appears to contain 9 entries; there are
+      // some pointers immediately after. So probably if a card specified A-F,
+      // its range would be filled in with garbage in the original game.
+      default:
+        throw runtime_error("invalid fixed range index");
+    }
+  }
+}
+
+string name_for_rarity(uint8_t rarity) {
+  static const vector<const char*> names({
+    "N1",
+    "R1",
+    "S",
+    "E",
+    "N2",
+    "N3",
+    "N4",
+    "R2",
+    "R3",
+    "R4",
+    "SS",
+    "D1",
+    "D2",
+    "INVIS",
+  });
+  try {
+    return names.at(rarity - 1);
+  } catch (const out_of_range&) {
+    return string_printf("(%02hhX)", rarity);
+  }
+}
+
+string name_for_target_mode(uint8_t target_mode) {
+  static const vector<const char*> names({
+    "NONE",
+    "SINGLE",
+    "MULTI",
+    "SELF",
+    "TEAM",
+    "ALL",
+    "MULTI-ALLY",
+    "ALL-ALLY",
+    "ALL-ATTACK",
+    "OWN-FCS",
+  });
+  try {
+    return names.at(target_mode);
+  } catch (const out_of_range&) {
+    return string_printf("(%02hhX)", target_mode);
+  }
+}
+
+string string_for_colors(const parray<uint8_t, 8>& colors) {
+  string ret;
+  for (size_t x = 0; x < 8; x++) {
+    if (colors[x]) {
+      ret += '0' + colors[x];
+    }
+  }
+  if (ret.empty()) {
+    return "none";
+  }
+  return ret;
+}
+
+string string_for_assist_turns(uint8_t turns) {
+  if (turns == 90) {
+    return "ONCE";
+  } else if (turns == 99) {
+    return "FOREVER";
+  } else {
+    return string_printf("%hhu", turns);
+  }
+}
+
+string string_for_range(const parray<be_uint32_t, 6>& range) {
+  string ret;
+  for (size_t x = 0; x < 6; x++) {
+    ret += string_printf("%05" PRIX32 "/", range[x].load());
+  }
+  while (starts_with(ret, "00000/")) {
+    ret = ret.substr(6);
+  }
+  if (!ret.empty()) {
+    ret.resize(ret.size() - 1);
+  }
+  return ret;
+}
+
+string Ep3CardStats::str() const {
+  string type_str;
+  try {
+    type_str = name_for_card_type.at(this->type);
+  } catch (const out_of_range&) {
+    type_str = string_printf("%02hhX", this->type);
+  }
+  string rarity_str = name_for_rarity(this->rarity);
+  string target_mode_str = name_for_target_mode(this->target_mode);
+  string range_str = string_for_range(this->range);
+  string assist_turns_str = string_for_assist_turns(this->assist_turns);
+  string hp_str = this->hp.str();
+  string ap_str = this->ap.str();
+  string tp_str = this->tp.str();
+  string mv_str = this->mv.str();
+  string left_str = string_for_colors(this->left_colors);
+  string right_str = string_for_colors(this->right_colors);
+  string top_str = string_for_colors(this->top_colors);
+  string effects_str;
+  for (size_t x = 0; x < 3; x++) {
+    if (this->effects[x].is_empty()) {
+      continue;
+    }
+    if (!effects_str.empty()) {
+      effects_str += ", ";
+    }
+    effects_str += this->effects[x].str();
+  }
+  return string_printf(
+      "[Card: %04" PRIX32 " name=%s type=%s-%02hhX rare=%s cost=%hhX+%hhX "
+      "target=%s range=%s assist_turns=%s cannot_move=%s cannot_attack=%s "
+      "hidden=%s hp=%s ap=%s tp=%s mv=%s left=%s right=%s top=%s a2=%08" PRIX32 " "
+      "assist_effect=[%hu, %hu] a3=[%hu, %hu] has_effects=%s effects=[%s]]",
+      this->card_id.load(),
+      this->name.data(),
+      type_str.c_str(),
+      this->subtype,
+      rarity_str.c_str(),
+      this->self_cost,
+      this->ally_cost,
+      target_mode_str.c_str(),
+      range_str.c_str(),
+      assist_turns_str.c_str(),
+      this->cannot_move ? "true" : "false",
+      this->cannot_attack ? "true" : "false",
+      this->hide_in_deck_edit ? "true" : "false",
+      hp_str.c_str(),
+      ap_str.c_str(),
+      tp_str.c_str(),
+      mv_str.c_str(),
+      left_str.c_str(),
+      right_str.c_str(),
+      top_str.c_str(),
+      this->unknown_a2.load(),
+      this->assist_effect[0].load(),
+      this->assist_effect[1].load(),
+      this->unknown_a3[0].load(),
+      this->unknown_a3[1].load(),
+      this->has_effects ? "true" : "false",
+      effects_str.c_str());
+}
+
+
+
+Ep3DataIndex::Ep3DataIndex(const string& directory) {
+  static constexpr bool debug_enabled = false;
+
+  unordered_map<uint32_t, vector<string>> card_tags;
+  if (debug_enabled) {
+    unordered_map<uint32_t, string> card_text;
+    try {
+      string data = prs_decompress(load_file(directory + "/cardtext.mnr"));
+      StringReader r(data);
+
+      while (!r.eof()) {
+        uint32_t card_id = stoul(r.get_cstr());
+
+        // Most cards have multiple pages, but we only care about the first page
+        // (for now)
+        string text = r.get_cstr();
+
+        // Preprocess text: first, delete all color markers
+        size_t offset = text.find("\tC");
+        while (offset != string::npos) {
+          text = text.substr(0, offset) + text.substr(offset + 3);
+          offset = text.find("\tC");
+        }
+        // Preprocess text: delete all initial lines that don't start with \t
+        offset = text.find('\t');
+        if (offset == string::npos) {
+          text.clear();
+        } else {
+          text = text.substr(offset);
+        }
+        // Preprocess text: merge lines that don't begin with \t
+        for (offset = 0; offset < text.size(); offset++) {
+          if (text[offset] == '\n' && text[offset + 1] != '\t') {
+            text = text.substr(0, offset) + text.substr(offset + 1);
+            offset--;
+          }
+        }
+
+        // Split text into tags
+        vector<string> tags;
+        auto lines = split(text, '\n');
+        for (const auto& line : lines) {
+          if (line[0] == '\t' && line[1] == 'D') {
+            tags.emplace_back("D: " + line.substr(2));
+          } else if (line[0] == '\t' && line[1] == 'S') {
+            tags.emplace_back("S: " + line.substr(2));
+          }
+        }
+
+        if (!card_text.emplace(card_id, move(text)).second) {
+          throw runtime_error("duplicate card text id");
+        }
+        if (!card_tags.emplace(card_id, move(tags)).second) {
+          throw logic_error("duplicate card tags id");
+        }
+
+        r.go((r.where() + 0x3FF) & (~0x3FF));
+      }
+
+    } catch (const exception& e) {
+      log(WARNING, "Failed to load card text: %s", e.what());
+    }
+  }
+
+  try {
+    this->compressed_card_definitions = load_file(directory + "/cardupdate.mnr");
+    string data = prs_decompress(this->compressed_card_definitions);
+    // There's a footer after the card definitions, but we ignore it
+    if (data.size() % sizeof(Ep3CardStats) != sizeof(Ep3CardStatsFooter)) {
+      throw runtime_error(string_printf(
+          "decompressed card update file size %zX is not aligned with card definition size %zX (%zX extra bytes)",
+          data.size(), sizeof(Ep3CardStats), data.size() % sizeof(Ep3CardStats)));
+    }
+    const auto* stats = reinterpret_cast<const Ep3CardStats*>(data.data());
+    size_t max_cards = data.size() / sizeof(Ep3CardStats);
+    for (size_t x = 0; x < max_cards; x++) {
+      // The last card entry has the build date and some other metadata (and
+      // isn't a real card, obviously), so skip it. Seems like the card ID is
+      // always a large number that won't fit in a uint16_t, so we use that to
+      // determine if the entry is a real card or not.
+      if (stats[x].card_id & 0xFFFF0000) {
+        continue;
+      }
+      shared_ptr<CardEntry> entry(new CardEntry({stats[x], {}}));
+      if (!this->card_definitions.emplace(entry->stats.card_id, entry).second) {
+        throw runtime_error(string_printf(
+            "duplicate card id: %08" PRIX32, entry->stats.card_id.load()));
+      }
+
+      entry->stats.hp.decode_code();
+      entry->stats.ap.decode_code();
+      entry->stats.tp.decode_code();
+      entry->stats.mv.decode_code();
+      entry->stats.decode_range();
+
+      if (debug_enabled) {
+        string card_str = entry->stats.str();
+        try {
+          string tags_str = join(card_tags.at(stats[x].card_id), ", ");
+          fprintf(stderr, "%s tags: [%s]\n", card_str.c_str(), tags_str.c_str());
+        } catch (const out_of_range&) {
+          fprintf(stderr, "%s\n", card_str.c_str());
+        }
+      }
+    }
+
+    log(INFO, "Indexed %zu Episode 3 card definitions", this->card_definitions.size());
+  } catch (const exception& e) {
+    log(WARNING, "Failed to load Episode 3 card update: %s", e.what());
+  }
+
+  for (const auto& filename : list_directory(directory)) {
+    try {
+      shared_ptr<MapEntry> entry;
+
+      if (ends_with(filename, ".mnmd")) {
+        entry.reset(new MapEntry(load_object_file<Ep3Map>(directory + "/" + filename)));
+      } else if (ends_with(filename, ".mnm")) {
+        entry.reset(new MapEntry(load_file(directory + "/" + filename)));
+      }
+
+      if (entry.get()) {
+        if (!this->maps.emplace(entry->map.map_number, entry).second) {
+          throw runtime_error("duplicate map number");
+        }
+        string name = entry->map.name;
+        log(INFO, "Indexed Episode 3 map %s (%08" PRIX32 "; %s)",
+            filename.c_str(), entry->map.map_number.load(), name.c_str());
+      }
+
+    } catch (const exception& e) {
+      log(WARNING, "Failed to index Episode 3 map %s: %s",
+          filename.c_str(), e.what());
+    }
+  }
+}
+
+Ep3DataIndex::MapEntry::MapEntry(const Ep3Map& map) : map(map) { }
+
+Ep3DataIndex::MapEntry::MapEntry(const string& compressed)
+  : compressed_data(compressed) {
+  string decompressed = prs_decompress(this->compressed_data);
+  if (decompressed.size() != sizeof(Ep3Map)) {
+    throw runtime_error(string_printf(
+        "decompressed data size is incorrect (expected %zu bytes, read %zu bytes)",
+        sizeof(Ep3Map), decompressed.size()));
+  }
+  this->map = *reinterpret_cast<const Ep3Map*>(decompressed.data());
+}
+
+string Ep3DataIndex::MapEntry::compressed() const {
+  if (this->compressed_data.empty()) {
+    this->compressed_data = prs_compress(&this->map, sizeof(this->map));
+  }
+  return this->compressed_data;
+}
+
+const string& Ep3DataIndex::get_compressed_card_definitions() const {
+  if (this->compressed_card_definitions.empty()) {
+    throw runtime_error("card definitions are not available");
+  }
+  return this->compressed_card_definitions;
+}
+
+shared_ptr<const Ep3DataIndex::CardEntry> Ep3DataIndex::get_card_definition(
+    uint32_t id) const {
+  return this->card_definitions.at(id);
+}
+
+const string& Ep3DataIndex::get_compressed_map_list() const {
+  if (this->compressed_map_list.empty()) {
+    // TODO: Write a version of prs_compress that takes iovecs (or something
+    // similar) so we can eliminate all this string copying here.
+    StringWriter entries_w;
+    StringWriter strings_w;
+
+    for (const auto& map_it : this->maps) {
+      Ep3MapList::Entry e;
+      const auto& map = map_it.second->map;
+      e.map_x = map.map_x;
+      e.map_y = map.map_y;
+      e.scene_data2 = map.scene_data2;
+      e.map_number = map.map_number.load();
+      e.width = map.width;
+      e.height = map.height;
+      e.map_tiles = map.map_tiles;
+      e.modification_tiles = map.modification_tiles;
+
+      e.name_offset = strings_w.size();
+      strings_w.write(map.name.data(), map.name.len());
+      strings_w.put_u8(0);
+      e.location_name_offset = strings_w.size();
+      strings_w.write(map.location_name.data(), map.location_name.len());
+      strings_w.put_u8(0);
+      e.quest_name_offset = strings_w.size();
+      strings_w.write(map.quest_name.data(), map.quest_name.len());
+      strings_w.put_u8(0);
+      e.description_offset = strings_w.size();
+      strings_w.write(map.description.data(), map.description.len());
+      strings_w.put_u8(0);
+
+      e.unknown_a2 = 0xFF000000;
+
+      entries_w.put(e);
+    }
+
+    Ep3MapList header;
+    header.num_maps = this->maps.size();
+    header.unknown_a1 = 0;
+    header.strings_offset = entries_w.size();
+    header.total_size = sizeof(Ep3MapList) + entries_w.size() + strings_w.size();
+
+    StringWriter w;
+    w.put(header);
+    w.write(entries_w.str());
+    w.write(strings_w.str());
+
+    StringWriter compressed_w;
+    compressed_w.put_u32b(w.str().size());
+    compressed_w.write(prs_compress(w.str()));
+    this->compressed_map_list = move(compressed_w.str());
+    log(INFO, "Generated Episode 3 compressed map list (%zu -> %zu bytes)",
+        w.size(), this->compressed_map_list.size());
+  }
+  return this->compressed_map_list;
+}
+
+shared_ptr<const Ep3DataIndex::MapEntry> Ep3DataIndex::get_map(uint32_t id) const {
+  return this->maps.at(id);
+}

src/Episode3.hh

@@ -0,0 +1,351 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <string>
+#include <map>
+#include <memory>
+#include <unordered_map>
+#include <phosg/Encoding.hh>
+
+#include "Text.hh"
+
+
+
+// Note: Much of the structures and enums here are based on the card list file,
+// and comparing the card text with the data in the file. Some inferences may be
+// incorrect here, since Episode 3's card text is wrong in various places.
+
+struct Ep3CardStats {
+  enum Rarity : uint8_t {
+    N1    = 0x01,
+    R1    = 0x02,
+    S     = 0x03,
+    E     = 0x04,
+    N2    = 0x05,
+    N3    = 0x06,
+    N4    = 0x07,
+    R2    = 0x08,
+    R3    = 0x09,
+    R4    = 0x0A,
+    SS    = 0x0B,
+    D1    = 0x0C,
+    D2    = 0x0D,
+    INVIS = 0x0E,
+  };
+
+  enum Type : uint8_t {
+    SC_HUNTERS = 0x00, // No subtypes
+    SC_ARKZ    = 0x01, // No subtypes
+    ITEM       = 0x02, // Subtype 01 = sword, 02 = gun, 03 = cane. TODO: there are many more subtypes than those 3
+    CREATURE   = 0x03, // No subtypes (TODO: Where are attributes stored then?)
+    ACTION     = 0x04, // TODO: What do the subtypes mean? Are they actually flags instead?
+    ASSIST     = 0x05, // No subtypes
+  };
+
+  struct Stat {
+    enum Type : uint8_t {
+      BLANK = 0,
+      STAT = 1,
+      PLUS_STAT = 2,
+      MINUS_STAT = 3,
+      EQUALS_STAT = 4,
+      UNKNOWN = 5,
+      PLUS_UNKNOWN = 6,
+      MINUS_UNKNOWN = 7,
+      EQUALS_UNKNOWN = 8,
+    };
+    be_uint16_t code;
+    Type type;
+    int8_t stat;
+
+    void decode_code();
+    std::string str() const;
+  } __attribute__((packed));
+
+  struct Effect {
+    uint8_t command;
+    ptext<char, 0x0F> expr; // May be blank if the command doesn't use it
+    uint8_t when;
+    ptext<char, 4> arg1;
+    ptext<char, 4> arg2;
+    ptext<char, 4> arg3;
+    parray<uint8_t, 3> unknown_a3;
+
+    bool is_empty() const;
+    static std::string str_for_arg(const std::string& arg);
+    std::string str() const;
+  } __attribute__((packed));
+
+  be_uint32_t card_id;
+  parray<uint8_t, 0x40> jp_name;
+  int8_t type; // Type enum. If <0, then this is the end of the card list
+  uint8_t self_cost; // ATK dice points required
+  uint8_t ally_cost; // ATK points from allies required; PBs use this
+  uint8_t unused_a0; // Always 0
+  Stat hp;
+  Stat ap;
+  Stat tp;
+  Stat mv;
+  parray<uint8_t, 8> left_colors;
+  parray<uint8_t, 8> right_colors;
+  parray<uint8_t, 8> top_colors;
+  parray<be_uint32_t, 6> range;
+  be_uint32_t unused_a1; // Always 0
+  // Target modes:
+  // 00 = no targeting (used for defense cards, mags, shields, etc.)
+  // 01 = single enemy
+  // 02 = multiple enemies (with range)
+  // 03 = self (assist)
+  // 04 = team (assist)
+  // 05 = everyone (assist)
+  // 06 = multiple allies (with range); only used by Shifta
+  // 07 = all allies including yourself; see Anti, Resta, Leilla
+  // 08 = all (attack); see e.g. Last Judgment, Earthquake
+  // 09 = your own FCs but not SCs; see Traitor
+  uint8_t target_mode;
+  uint8_t assist_turns; // 90 (dec) = once, 99 (dec) = forever
+  uint8_t cannot_move; // 0 for SC and creature cards; 1 for everything else
+  uint8_t cannot_attack; // 1 for shields, mags, defense actions, and assist cards
+  uint8_t unused_a2; // Always 0
+  uint8_t hide_in_deck_edit; // 0 = player can use this card (appears in deck edit)
+  uint8_t subtype; // e.g. gun, sword, etc. (used for checking if SCs can use it)
+  uint8_t rarity; // Rarity enum
+  be_uint32_t unknown_a2;
+  // These two fields seem to always contain the same value, and are always 0
+  // for non-assist cards and nonzero for assists. Each assist card has a unique
+  // value here and no effects, which makes it look like this is how assist
+  // effects are implemented. There seems to be some 1k-modulation going on here
+  // too; most cards are in the range 101-174 but a few have e.g. 1150, 2141. A
+  // few pairs of cards have the same effect, which makes it look like some
+  // other fields are also involved in determining their effects (see e.g. Skip
+  // Draw / Skip Move, Dice Fever / Dice Fever +, Reverse Card / Rich +).
+  parray<be_uint16_t, 2> assist_effect;
+  parray<be_uint16_t, 2> unknown_a3;
+  ptext<char, 0x14> name;
+  ptext<char, 0x0B> jp_short_name;
+  ptext<char, 0x07> short_name;
+  be_uint16_t has_effects; // 1 if any of the following structs are not blank
+  Effect effects[3];
+
+  void decode_range();
+  std::string str() const;
+} __attribute__((packed)); // 0x128 bytes in total
+
+struct Ep3CardStatsFooter {
+  be_uint32_t num_cards1;
+  be_uint32_t unknown_a1;
+  be_uint32_t num_cards2;
+  be_uint32_t unknown_a2[11];
+  be_uint32_t unknown_offset_a3;
+  be_uint32_t unknown_a4[3];
+  be_uint32_t footer_offset;
+  be_uint32_t unknown_a5[3];
+} __attribute__((packed));
+
+struct Ep3Deck {
+  ptext<char, 0x10> name;
+  be_uint32_t client_id; // 0-3
+  // List of card IDs. The card count is the number of nonzero entries here
+  // before a zero entry (or 50 if no entries are nonzero). The first card ID is
+  // the SC card, which the game implicitly subtracts from the limit - so a
+  // valid deck should actually have 31 cards in it.
+  parray<le_uint16_t, 50> card_ids;
+  be_uint32_t unknown_a1;
+  // Last modification time
+  le_uint16_t year;
+  uint8_t month;
+  uint8_t day;
+  uint8_t hour;
+  uint8_t minute;
+  uint8_t second;
+  uint8_t unknown_a2;
+} __attribute__((packed)); // 0x84 bytes in total
+
+struct Ep3Config {
+  // Offsets in comments in this struct are relative to start of 61/98 command
+  /* 0728 */ parray<uint8_t, 0x1434> unknown_a1;
+  /* 1B5C */ parray<Ep3Deck, 25> decks;
+  /* 2840 */ uint64_t unknown_a2;
+  /* 2848 */ be_uint32_t offline_clv_exp; // CLvOff = this / 100
+  /* 284C */ be_uint32_t online_clv_exp; // CLvOn = this / 100
+  /* 2850 */ parray<uint8_t, 0x14C> unknown_a3;
+  /* 299C */ ptext<char, 0x10> name;
+  // Other records are probably somewhere in here - e.g. win/loss, play time, etc.
+  /* 29AC */ parray<uint8_t, 0xCC> unknown_a4;
+} __attribute__((packed));
+
+struct Ep3BattleRules {
+  // When this structure is used in a map/quest definition, FF in any of these
+  // fields means the user is allowed to override it. Any non-FF fields are
+  // fixed for the map/quest and cannot be overridden.
+  uint8_t overall_time_limit; // In increments of 5 minutes; 0 = unlimited
+  uint8_t phase_time_limit; // In seconds; 0 = unlimited
+  uint8_t allowed_cards; // 0 = any, 1 = N-rank only, 2 = N and R, 3 = N, R, and S 
+  uint8_t min_dice; // 0 = default (1)
+  // 4
+  uint8_t max_dice; // 0 = default (6)
+  uint8_t disable_deck_shuffle; // 0 = shuffle on, 1 = off
+  uint8_t disable_deck_loop; // 0 = loop on, 1 = off
+  uint8_t char_hp;
+  // 8
+  uint8_t hp_type; // 0 = defeat player, 1 = defeat team, 2 = common hp
+  uint8_t no_assist_cards; // 1 = assist cards disallowed
+  uint8_t disable_dialogue; // 0 = dialogue on, 1 = dialogue off
+  uint8_t dice_exchange_mode; // 0 = high attack, 1 = high defense, 2 = none
+  // C
+  uint8_t disable_dice_boost; // 0 = dice boost on, 1 = off
+  parray<uint8_t, 3> unused;
+} __attribute__((packed));
+
+
+
+struct Ep3MapList {
+  be_uint32_t num_maps;
+  be_uint32_t unknown_a1; // Always 0?
+  be_uint32_t strings_offset; // From after total_size field (add 0x10 to this value)
+  be_uint32_t total_size; // Including header, entries, and strings
+
+  struct Entry { // Should be 0x220 bytes in total
+    // These 3 fields probably include the location ID (scenery to load) and the
+    // music ID
+    be_uint16_t map_x;
+    be_uint16_t map_y;
+    be_uint16_t scene_data2;
+    be_uint16_t map_number;
+    // Text offsets are from the beginning of the strings block after all map
+    // entries (that is, add strings_offset to them to get the string offset)
+    be_uint32_t name_offset;
+    be_uint32_t location_name_offset;
+    be_uint32_t quest_name_offset;
+    be_uint32_t description_offset;
+    be_uint16_t width;
+    be_uint16_t height;
+    parray<uint8_t, 0x100> map_tiles;
+    parray<uint8_t, 0x100> modification_tiles;
+    be_uint32_t unknown_a2; // Seems to always be 0xFF000000
+  } __attribute__((packed));
+
+  // Variable-length fields:
+  // Entry entries[num_maps];
+  // char strings[...EOF]; // Null-terminated strings, pointed to by offsets in Entry structs
+} __attribute__((packed));
+
+struct Ep3CompressedMapHeader { // .mnm file format
+  le_uint32_t map_number;
+  le_uint32_t compressed_data_size;
+  // Compressed data immediately follows (which decompresses to an Ep3Map)
+} __attribute__((packed));
+
+struct Ep3Map { // .mnmd format
+  /* 0000 */ be_uint32_t unknown_a1;
+  /* 0004 */ be_uint32_t map_number;
+  /* 0008 */ uint8_t width;
+  /* 0009 */ uint8_t height;
+  /* 000A */ uint8_t scene_data2; // TODO: What is this?
+  // All alt_maps fields (including the floats) past num_alt_maps are filled in
+  // with FF. For example, if num_alt_maps == 8, the last two fields in each
+  // alt_maps array are filled with FF.
+  /* 000B */ uint8_t num_alt_maps; // TODO: What are the alt maps for?
+  // In the map_tiles array, the values are:
+  // 00 = not a valid tile
+  // 01 = valid tile unless punched out (later)
+  // 02 = team A start (1v1)
+  // 03, 04 = team A start (2v2)
+  // 05 = ???
+  // 06, 07 = team B start (2v2)
+  // 08 = team B start (1v1)
+  // Note that the game displays the map reversed vertically in the preview
+  // window. For example, player 1 is on team A, which usually starts at the top
+  // of the map as defined in this struct, or at the bottom as shown in the
+  // preview window.
+  /* 000C */ parray<uint8_t, 0x100> map_tiles;
+  /* 010C */ parray<uint8_t, 0x0C> unknown_a2;
+  /* 0118 */ parray<uint8_t, 0x100> alt_maps1[0x0A];
+  /* 0B18 */ parray<uint8_t, 0x100> alt_maps2[0x0A];
+  /* 1518 */ parray<be_float, 0x12> alt_maps_unknown_a3[0x0A];
+  /* 17E8 */ parray<be_float, 0x12> alt_maps_unknown_a4[0x0A];
+  /* 1AB8 */ parray<be_float, 0x6C> unknown_a5;
+  // In the modification_tiles array, the values are:
+  // 10 = blocked (as if the corresponding map_tiles value was 00)
+  // 20 = blocked (maybe one of 10 or 20 are passable by Aerial characters though)
+  // 30, 31 = teleporters (green, red)
+  // 40-44 = ???? (used in 244, 2E4, 2F9)
+  // 50 = appears as improperly-z-buffered teal cube in preview
+  // TODO: There may be more values that are valid here.
+  /* 1C68 */ parray<uint8_t, 0x100> modification_tiles;
+  /* 1D68 */ parray<uint8_t, 0x74> unknown_a6;
+  /* 1DDC */ Ep3BattleRules default_rules;
+  /* 1DEC */ parray<uint8_t, 4> unknown_a7;
+  /* 1DF0 */ ptext<char, 0x14> name;
+  /* 1E04 */ ptext<char, 0x14> location_name;
+  /* 1E18 */ ptext<char, 0x3C> quest_name; // == location_name if not a quest
+  /* 1E54 */ ptext<char, 0x190> description;
+  /* 1FE4 */ be_uint16_t map_x;
+  /* 1FE6 */ be_uint16_t map_y;
+  struct NPCDeck {
+    ptext<char, 0x18> name;
+    parray<be_uint16_t, 0x20> card_ids; // Last one appears to always be FFFF
+  } __attribute__((packed));
+  /* 1FE8 */ NPCDeck npc_decks[3]; // Unused if name[0] == 0
+  struct NPCCharacter {
+    parray<be_uint16_t, 2> unknown_a1;
+    parray<uint8_t, 4> unknown_a2;
+    ptext<char, 0x10> name;
+    parray<be_uint16_t, 0x7E> unknown_a3;
+  } __attribute__((packed));
+  /* 20F0 */ NPCCharacter npc_chars[3]; // Unused if name[0] == 0
+  /* 242C */ parray<uint8_t, 0x14> unknown_a8; // Always FF?
+  /* 2440 */ ptext<char, 0x190> before_message;
+  /* 25D0 */ ptext<char, 0x190> after_message;
+  /* 2760 */ ptext<char, 0x190> dispatch_message; // Usually "You can only dispatch <character>" or blank
+  struct DialogueSet {
+    be_uint16_t unknown_a1;
+    be_uint16_t unknown_a2; // Always 0x0064 if valid, 0xFFFF if unused?
+    ptext<char, 0x40> strings[4];
+  } __attribute__((packed)); // Total size: 0x104 bytes
+  /* 28F0 */ DialogueSet dialogue_sets[3][0x10]; // Up to 0x10 per valid NPC
+  /* 59B0 */ be_uint16_t reward_card_id; // TODO: This could be an array. The only examples I've seen have only one here
+  /* 59B2 */ parray<be_uint16_t, 0x33> unknown_a9;
+  /* 5A18 */
+} __attribute__((packed));
+
+class Ep3DataIndex {
+public:
+  explicit Ep3DataIndex(const std::string& directory);
+
+  struct CardEntry {
+    Ep3CardStats stats;
+    std::vector<std::string> text;
+  };
+
+  class MapEntry {
+  public:
+    Ep3Map map;
+
+    MapEntry(const Ep3Map& map);
+    MapEntry(const std::string& compressed_data);
+
+    std::string compressed() const;
+
+  private:
+    mutable std::string compressed_data;
+  };
+
+  const std::string& get_compressed_card_definitions() const;
+  std::shared_ptr<const CardEntry> get_card_definition(uint32_t id) const;
+
+  const std::string& get_compressed_map_list() const;
+  std::shared_ptr<const MapEntry> get_map(uint32_t id) const;
+
+private:
+  std::string compressed_card_definitions;
+  std::unordered_map<uint32_t, std::shared_ptr<CardEntry>> card_definitions;
+
+  // The compressed map list is generated on demand from the maps map below.
+  // It's marked mutable because the logical consistency of the Ep3DataIndex
+  // object is not violated from the caller's perspective even if we don't
+  // generate the compressed map list at load time.
+  mutable std::string compressed_map_list;
+  std::map<uint32_t, std::shared_ptr<MapEntry>> maps;
+};

src/FunctionCompiler.cc

@@ -0,0 +1,231 @@
+#include "FunctionCompiler.hh"
+
+#include <stdio.h>
+#include <string.h>
+
+#include <stdexcept>
+#include <phosg/Filesystem.hh>
+
+#ifdef HAVE_RESOURCE_FILE
+#include <resource_file/Emulators/PPC32Emulator.hh>
+#endif
+
+#include "CommandFormats.hh"
+
+using namespace std;
+
+
+
+bool function_compiler_available() {
+#ifndef HAVE_RESOURCE_FILE
+  return false;
+#else
+  return true;
+#endif
+}
+
+
+
+string CompiledFunctionCode::generate_client_command(
+      const unordered_map<string, uint32_t>& label_writes,
+      const string& suffix) const {
+  S_ExecuteCode_Footer_GC_B2 footer;
+  footer.num_relocations = this->relocation_deltas.size();
+  footer.unused1.clear();
+  footer.entrypoint_addr_offset = this->entrypoint_offset_offset;
+  footer.unused2.clear();
+
+  StringWriter w;
+  if (!label_writes.empty()) {
+    string modified_code = this->code;
+    for (const auto& it : label_writes) {
+      size_t offset = this->label_offsets.at(it.first);
+      if (offset > modified_code.size() - 4) {
+        throw runtime_error("label out of range");
+      }
+      *reinterpret_cast<be_uint32_t*>(modified_code.data() + offset) = it.second;
+    }
+    w.write(modified_code);
+  } else {
+    w.write(this->code);
+  }
+  w.write(suffix);
+  while (w.size() & 3) {
+    w.put_u8(0);
+  }
+
+  footer.relocations_offset = w.size();
+  for (uint16_t delta : this->relocation_deltas) {
+    w.put_u16b(delta);
+  }
+  if (this->relocation_deltas.size() & 1) {
+    w.put_u16(0);
+  }
+
+  w.put(footer);
+  return move(w.str());
+}
+
+shared_ptr<CompiledFunctionCode> compile_function_code(
+    const string& directory, const string& name, const string& text) {
+#ifndef HAVE_RESOURCE_FILE
+  (void)directory;
+  (void)name;
+  (void)text;
+  throw runtime_error("PowerPC assembler is not available");
+
+#else
+  std::unordered_set<string> get_include_stack; // For mutual recursion detection
+  function<string(const string&)> get_include = [&](const string& name) -> string {
+    if (!get_include_stack.emplace(name).second) {
+      throw runtime_error("mutual recursion between includes");
+    }
+
+    string filename = directory + "/" + name + ".inc.s";
+    if (isfile(filename)) {
+      return PPC32Emulator::assemble(load_file(filename), get_include).code;
+    }
+    filename = directory + "/" + name + ".inc.bin";
+    if (isfile(filename)) {
+      return load_file(filename);
+    }
+    throw runtime_error("data not found for include " + name);
+  };
+
+  shared_ptr<CompiledFunctionCode> ret(new CompiledFunctionCode());
+  ret->name = name;
+  ret->index = 0;
+
+  auto assembled = PPC32Emulator::assemble(text, get_include);
+  ret->code = move(assembled.code);
+  ret->label_offsets = move(assembled.label_offsets);
+
+  set<uint32_t> reloc_indexes;
+  for (const auto& it : ret->label_offsets) {
+    if (starts_with(it.first, "reloc")) {
+      reloc_indexes.emplace(it.second / 4);
+    } else if (starts_with(it.first, "newserv_index_")) {
+      ret->index = stoul(it.first.substr(14), nullptr, 16);
+    }
+  }
+
+  try {
+    ret->entrypoint_offset_offset = ret->label_offsets.at("entry_ptr");
+  } catch (const out_of_range&) {
+    throw runtime_error("code does not contain entry_ptr label");
+  }
+
+  uint32_t prev_index = 0;
+  for (const auto& it : reloc_indexes) {
+    uint32_t delta = it - prev_index;
+    if (delta > 0xFFFF) {
+      throw runtime_error("relocation delta too far away");
+    }
+    ret->relocation_deltas.emplace_back(delta);
+    prev_index = it;
+  }
+
+  return ret;
+#endif
+}
+
+
+
+FunctionCodeIndex::FunctionCodeIndex(const string& directory) {
+  if (!function_compiler_available()) {
+    log(INFO, "Function compiler is not available");
+    return;
+  }
+
+  uint32_t next_menu_item_id = 0;
+  for (const auto& filename : list_directory(directory)) {
+    if (!ends_with(filename, ".s") || ends_with(filename, ".inc.s")) {
+      continue;
+    }
+    bool is_patch = ends_with(filename, ".patch.s");
+    string name = filename.substr(0, filename.size() - (is_patch ? 8 : 2));
+
+    try {
+      string path = directory + "/" + filename;
+      string text = load_file(path);
+      auto code = compile_function_code(directory, name, text);
+      if (code->index != 0) {
+        if (!this->index_to_function.emplace(code->index, code).second) {
+          throw runtime_error(string_printf(
+              "duplicate function index: %08" PRIX32, code->index));
+        }
+      }
+      this->name_to_function.emplace(name, code);
+      if (is_patch) {
+        this->menu_item_id_to_patch_function.emplace(next_menu_item_id++, code);
+        this->name_to_patch_function.emplace(name, code);
+      }
+      if (code->index) {
+        log(INFO, "Compiled function %02X => %s", code->index, name.c_str());
+      } else {
+        log(INFO, "Compiled function %s", name.c_str());
+      }
+
+    } catch (const exception& e) {
+      log(WARNING, "Failed to compile function %s: %s", name.c_str(), e.what());
+    }
+  }
+}
+
+vector<MenuItem> FunctionCodeIndex::patch_menu() const {
+  vector<MenuItem> ret;
+  ret.emplace_back(PatchesMenuItemID::GO_BACK, u"Go back", u"", 0);
+  for (const auto& it : this->name_to_patch_function) {
+    const auto& fn = it.second;
+    ret.emplace_back(fn->menu_item_id, decode_sjis(fn->name), u"",
+        MenuItem::Flag::REQUIRES_SEND_FUNCTION_CALL);
+  }
+  return ret;
+}
+
+
+
+DOLFileIndex::DOLFileIndex(const string& directory) {
+  if (!function_compiler_available()) {
+    log(INFO, "Function compiler is not available");
+    return;
+  }
+  if (!isdir(directory)) {
+    log(INFO, "DOL file directory is missing");
+    return;
+  }
+
+  uint32_t next_menu_item_id = 0;
+  for (const auto& filename : list_directory(directory)) {
+    if (!ends_with(filename, ".dol")) {
+      continue;
+    }
+    string name = filename.substr(0, filename.size() - 4);
+
+    try {
+      shared_ptr<DOLFile> dol(new DOLFile());
+      dol->menu_item_id = next_menu_item_id++;
+      dol->name = name;
+
+      string path = directory + "/" + filename;
+      dol->data = load_file(path);
+
+      this->name_to_file.emplace(dol->name, dol);
+      this->item_id_to_file.emplace_back(dol);
+      log(INFO, "Loaded DOL file %s", filename.c_str());
+
+    } catch (const exception& e) {
+      log(WARNING, "Failed to load DOL file %s: %s", filename.c_str(), e.what());
+    }
+  }
+}
+
+vector<MenuItem> DOLFileIndex::menu() const {
+  vector<MenuItem> ret;
+  ret.emplace_back(ProgramsMenuItemID::GO_BACK, u"Go back", u"", 0);
+  for (const auto& dol : this->item_id_to_file) {
+    ret.emplace_back(dol->menu_item_id, decode_sjis(dol->name), u"",
+        MenuItem::Flag::REQUIRES_SEND_FUNCTION_CALL);
+  }
+  return ret;
+}

src/FunctionCompiler.hh

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <inttypes.h>
+
+#include <string>
+#include <unordered_map>
+#include <map>
+#include <vector>
+#include <memory>
+
+#include "Menu.hh"
+
+
+
+bool function_compiler_available();
+
+
+
+// TODO: Support x86 function calls in the future. Currently we only support
+// PPC32 because I haven't written an appropriate x86 assembler yet.
+
+struct CompiledFunctionCode {
+  std::string code;
+  std::vector<uint16_t> relocation_deltas;
+  std::unordered_map<std::string, uint32_t> label_offsets;
+  uint32_t entrypoint_offset_offset;
+  std::string name;
+  uint32_t index; // 0 = unused (not registered in index_to_function)
+  uint32_t menu_item_id;
+
+  std::string generate_client_command(
+      const std::unordered_map<std::string, uint32_t>& label_writes = {},
+      const std::string& suffix = "") const;
+};
+
+std::shared_ptr<CompiledFunctionCode> compile_function_code(
+    const std::string& directory,
+    const std::string& name,
+    const std::string& text);
+
+
+
+struct FunctionCodeIndex {
+  FunctionCodeIndex(const std::string& directory);
+
+  std::unordered_map<std::string, std::shared_ptr<CompiledFunctionCode>> name_to_function;
+  std::unordered_map<uint32_t, std::shared_ptr<CompiledFunctionCode>> index_to_function;
+  std::unordered_map<uint32_t, std::shared_ptr<CompiledFunctionCode>> menu_item_id_to_patch_function;
+
+  std::map<std::string, std::shared_ptr<CompiledFunctionCode>> name_to_patch_function;
+
+  std::vector<MenuItem> patch_menu() const;
+  inline bool patch_menu_empty() const {
+    return this->name_to_patch_function.empty();
+  }
+};
+
+
+
+struct DOLFileIndex {
+  struct DOLFile {
+    uint32_t menu_item_id;
+    std::string name;
+    std::string data;
+  };
+
+  std::vector<std::shared_ptr<DOLFile>> item_id_to_file;
+  std::map<std::string, std::shared_ptr<DOLFile>> name_to_file;
+
+  DOLFileIndex(const std::string& directory);
+
+  std::vector<MenuItem> menu() const;
+  inline bool empty() const {
+    return this->name_to_file.empty() && this->item_id_to_file.empty();
+  }
+};

src/IPStackSimulator.cc

@@ -77,10 +77,8 @@ string IPStackSimulator::str_for_tcp_connection(shared_ptr<const IPClient> c,
 
 IPStackSimulator::IPStackSimulator(
     std::shared_ptr<struct event_base> base,
-    std::shared_ptr<Server> game_server,
     std::shared_ptr<ServerState> state)
   : base(base),
-    game_server(game_server),
     state(state),
     pcap_text_log_file(state->ip_stack_debug ? fopen("IPStackSimulator-Log.txt", "wt") : nullptr) {
   memset(this->host_mac_address_bytes, 0x90, 6);
@@ -779,9 +777,9 @@ void IPStackSimulator::open_server_connection(
       this->log(INFO, "Connected TCP connection %s to proxy server",
           conn_str.c_str());
     }
-  } else if (this->game_server.get()) {
-    this->game_server->connect_client(bevs[1], c->ipv4_addr, conn.client_port,
-        port_config->version, port_config->behavior);
+  } else if (this->state->game_server.get()) {
+    this->state->game_server->connect_client(bevs[1], c->ipv4_addr,
+        conn.client_port, port_config->version, port_config->behavior);
     this->log(INFO, "Connected TCP connection %s to game server",
         conn_str.c_str());
   } else {

src/IPStackSimulator.hh

@@ -17,7 +17,6 @@ class IPStackSimulator {
 public:
   IPStackSimulator(
       std::shared_ptr<struct event_base> base,
-      std::shared_ptr<Server> game_server,
       std::shared_ptr<ServerState> state);
   ~IPStackSimulator();
 
@@ -31,7 +30,6 @@ public:
 private:
   static PrefixedLogger log;
   std::shared_ptr<struct event_base> base;
-  std::shared_ptr<Server> game_server;
   std::shared_ptr<ServerState> state;
 
   using unique_listener = std::unique_ptr<struct evconnlistener, void(*)(struct evconnlistener*)>;

src/Items.cc

@@ -456,7 +456,7 @@ ItemData CommonItemCreator::create_drop_item(bool is_box, uint8_t episode,
 
     case 0x07: // meseta
       item.data1[0] = 0x04;
-      item.data2d = (90 * difficulty) + (random_int(0, 20) * (area * 2)); // meseta amount
+      item.data2d = (90 * difficulty) + (random_int(1, 20) * (area * 2)); // meseta amount
       break;
 
     default:

src/Lobby.cc

@@ -12,12 +12,12 @@ using namespace std;
 
 
 Lobby::Lobby() : lobby_id(0), min_level(0), max_level(0xFFFFFFFF),
-    next_game_item_id(0), version(GameVersion::GC), section_id(0), episode(1),
-    difficulty(0), mode(0), rare_seed(random_object<uint32_t>()), event(0),
-    block(0), type(0), leader_id(0), max_clients(12), flags(0) {
+    next_game_item_id(0x00810000), version(GameVersion::GC), section_id(0),
+    episode(1), difficulty(0), mode(0), rare_seed(random_object<uint32_t>()),
+    event(0), block(0), type(0), leader_id(0), max_clients(12), flags(0) {
 
   for (size_t x = 0; x < 12; x++) {
-    this->next_item_id[x] = 0;
+    this->next_item_id[x] = 0x00010000 + 0x00200000 * x;
   }
   this->next_drop_item = PlayerInventoryItem();
 }
@@ -57,9 +57,9 @@ size_t Lobby::count_clients() const {
   return ret;
 }
 
-void Lobby::add_client(shared_ptr<Client> c, bool reverse_indexes) {
+void Lobby::add_client(shared_ptr<Client> c) {
   ssize_t index;
-  if (reverse_indexes) {
+  if (c->prefer_high_lobby_client_id) {
     for (index = max_clients - 1; index >= 0; index--) {
       if (!this->clients[index].get()) {
         this->clients[index] = c;
@@ -84,8 +84,8 @@ void Lobby::add_client(shared_ptr<Client> c, bool reverse_indexes) {
   c->lobby_client_id = index;
   c->lobby_id = this->lobby_id;
 
-  // if there's no one else in the lobby, set the leader id as well
-  if (index == (max_clients - 1) * reverse_indexes) {
+  // If there's no one else in the lobby, set the leader id as well
+  if (index == (max_clients - 1) * c->prefer_high_lobby_client_id) {
     for (index = 0; index < max_clients; index++) {
       if (this->clients[index].get() && this->clients[index] != c) {
         break;
@@ -96,12 +96,13 @@ void Lobby::add_client(shared_ptr<Client> c, bool reverse_indexes) {
     }
   }
 
-  // If the lobby is a game, assign the inventory's item IDs
-  if (this->is_game()) {
+  // If the lobby is a game and item tracking is enabled, assign the inventory's
+  // item IDs
+  if (this->is_game() && (this->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
     auto& inv = c->game_data.player()->inventory;
-    size_t count = max<uint8_t>(inv.num_items, 30);
+    size_t count = min<uint8_t>(inv.num_items, 30);
     for (size_t x = 0; x < count; x++) {
-      inv.items[x].data.id = 0x00010000 + 0x00200000 * c->lobby_client_id + x;
+      inv.items[x].data.id = this->generate_item_id(c->lobby_client_id);
     }
     c->game_data.player()->print_inventory(stderr);
   }
@@ -118,7 +119,7 @@ void Lobby::remove_client(shared_ptr<Client> c) {
 
   this->clients[c->lobby_client_id] = nullptr;
 
-  // unassign the client's lobby if it matches the current lobby's id (it may
+  // Unassign the client's lobby if it matches the current lobby's id (it may
   // not match if the client was already added to another lobby - this can
   // happen during the lobby change procedure)
   if (c->lobby_id == this->lobby_id) {

src/Lobby.hh

@@ -18,14 +18,19 @@
 
 struct Lobby {
   enum Flag {
-    GAME = 0x01,
-    CHEATS_ENABLED = 0x02, // game only
-    PUBLIC = 0x04, // lobby only
-    EPISODE_3_ONLY = 0x08, // lobby & game
-    QUEST_IN_PROGRESS = 0x10, // game only
-    JOINABLE_QUEST_IN_PROGRESS = 0x20, // game only
-    DEFAULT = 0x40, // lobby only; not set for games and private lobbies
-    PERSISTENT = 0x80, // if not set, lobby is deleted when empty
+    GAME                       = 0x00000001,
+    EPISODE_3_ONLY             = 0x00000002,
+
+    // Flags used only for games
+    CHEATS_ENABLED             = 0x00000100,
+    QUEST_IN_PROGRESS          = 0x00000200,
+    JOINABLE_QUEST_IN_PROGRESS = 0x00000400,
+    ITEM_TRACKING_ENABLED      = 0x00000800,
+
+    // Flags used only for lobbies
+    PUBLIC                     = 0x00010000,
+    DEFAULT                    = 0x00020000,
+    PERSISTENT                 = 0x00040000,
   };
 
   uint32_t lobby_id;
@@ -80,7 +85,7 @@ struct Lobby {
   size_t count_clients() const;
   bool any_client_loading() const;
 
-  void add_client(std::shared_ptr<Client> c, bool reverse_indexes = true);
+  void add_client(std::shared_ptr<Client> c);
   void remove_client(std::shared_ptr<Client> c);
 
   void move_client_to_lobby(std::shared_ptr<Lobby> dest_lobby,

src/Main.cc

@@ -83,100 +83,6 @@ void populate_state_from_config(shared_ptr<ServerState> s,
   s->common_item_creator.reset(new CommonItemCreator(enemy_categories,
       box_categories, unit_types));
 
-  shared_ptr<vector<MenuItem>> information_menu_pc(new vector<MenuItem>());
-  shared_ptr<vector<MenuItem>> information_menu_gc(new vector<MenuItem>());
-  shared_ptr<vector<u16string>> information_contents(new vector<u16string>());
-
-  information_menu_gc->emplace_back(INFORMATION_MENU_GO_BACK, u"Go back",
-      u"Return to the\nmain menu", 0);
-  {
-    uint32_t item_id = 0;
-    for (const auto& item : d.at("InformationMenuContents")->as_list()) {
-      auto& v = item->as_list();
-      information_menu_pc->emplace_back(item_id, decode_sjis(v.at(0)->as_string()),
-          decode_sjis(v.at(1)->as_string()), 0);
-      information_menu_gc->emplace_back(item_id, decode_sjis(v.at(0)->as_string()),
-          decode_sjis(v.at(1)->as_string()), MenuItem::Flag::REQUIRES_MESSAGE_BOXES);
-      information_contents->emplace_back(decode_sjis(v.at(2)->as_string()));
-      item_id++;
-    }
-  }
-  s->information_menu_pc = information_menu_pc;
-  s->information_menu_gc = information_menu_gc;
-  s->information_contents = information_contents;
-
-  s->proxy_destinations_menu_pc.emplace_back(PROXY_DESTINATIONS_MENU_GO_BACK,
-      u"Go back", u"Return to the\nmain menu", 0);
-  s->proxy_destinations_menu_gc.emplace_back(PROXY_DESTINATIONS_MENU_GO_BACK,
-      u"Go back", u"Return to the\nmain menu", 0);
-  {
-    uint32_t item_id = 0;
-    for (const auto& item : d.at("ProxyDestinations-GC")->as_dict()) {
-      const string& netloc_str = item.second->as_string();
-      s->proxy_destinations_menu_gc.emplace_back(item_id, decode_sjis(item.first),
-          decode_sjis(netloc_str), 0);
-      s->proxy_destinations_gc.emplace_back(parse_netloc(netloc_str));
-      item_id++;
-    }
-  }
-  {
-    uint32_t item_id = 0;
-    for (const auto& item : d.at("ProxyDestinations-PC")->as_dict()) {
-      const string& netloc_str = item.second->as_string();
-      s->proxy_destinations_menu_pc.emplace_back(item_id, decode_sjis(item.first),
-          decode_sjis(netloc_str), 0);
-      s->proxy_destinations_pc.emplace_back(parse_netloc(netloc_str));
-      item_id++;
-    }
-  }
-  try {
-    const string& netloc_str = d.at("ProxyDestination-Patch")->as_string();
-    s->proxy_destination_patch = parse_netloc(netloc_str);
-    log(INFO, "Patch server proxy is enabled with destination %s", netloc_str.c_str());
-    for (auto& it : s->name_to_port_config) {
-      if (it.second->version == GameVersion::PATCH) {
-        it.second->behavior = ServerBehavior::PROXY_SERVER;
-      }
-    }
-  } catch (const out_of_range&) {
-    s->proxy_destination_patch.first = "";
-    s->proxy_destination_patch.second = 0;
-  }
-  try {
-    const string& netloc_str = d.at("ProxyDestination-BB")->as_string();
-    s->proxy_destination_bb = parse_netloc(netloc_str);
-    log(INFO, "BB proxy is enabled with destination %s", netloc_str.c_str());
-    for (auto& it : s->name_to_port_config) {
-      if (it.second->version == GameVersion::BB) {
-        it.second->behavior = ServerBehavior::PROXY_SERVER;
-      }
-    }
-  } catch (const out_of_range&) {
-    s->proxy_destination_bb.first = "";
-    s->proxy_destination_bb.second = 0;
-  }
-
-  s->main_menu.emplace_back(MAIN_MENU_GO_TO_LOBBY, u"Go to lobby",
-      u"Join the lobby", 0);
-  s->main_menu.emplace_back(MAIN_MENU_INFORMATION, u"Information",
-      u"View server information", MenuItem::Flag::REQUIRES_MESSAGE_BOXES);
-  if (!s->proxy_destinations_pc.empty()) {
-    s->main_menu.emplace_back(MAIN_MENU_PROXY_DESTINATIONS, u"Proxy server",
-        u"Connect to another\nserver", MenuItem::Flag::PC_ONLY);
-  }
-  if (!s->proxy_destinations_gc.empty()) {
-    s->main_menu.emplace_back(MAIN_MENU_PROXY_DESTINATIONS, u"Proxy server",
-        u"Connect to another\nserver", MenuItem::Flag::GC_ONLY);
-  }
-  s->main_menu.emplace_back(MAIN_MENU_DOWNLOAD_QUESTS, u"Download quests",
-      u"Download quests", 0);
-  s->main_menu.emplace_back(MAIN_MENU_DISCONNECT, u"Disconnect",
-      u"Disconnect", 0);
-
-  try {
-    s->welcome_message = decode_sjis(d.at("WelcomeMessage")->as_string());
-  } catch (const out_of_range&) { }
-
   auto local_address_str = d.at("LocalAddress")->as_string();
   try {
     s->local_address = s->all_addresses.at(local_address_str);
@@ -222,20 +128,19 @@ void populate_state_from_config(shared_ptr<ServerState> s,
     s->allow_unregistered_users = true;
   }
 
+  try {
+    s->item_tracking_enabled = d.at("EnableItemTracking")->as_bool();
+  } catch (const out_of_range&) {
+    s->item_tracking_enabled = true;
+  }
+
   for (const string& filename : list_directory("system/blueburst/keys")) {
     if (!ends_with(filename, ".nsk")) {
       continue;
     }
-    string contents = load_file("system/blueburst/keys/" + filename);
-    if (contents.size() != sizeof(PSOBBEncryption::KeyFile)) {
-      log(WARNING, "Blue Burst key file %s is the wrong size (%zu bytes; should be %zu bytes)",
-          filename.c_str(), contents.size(), sizeof(PSOBBEncryption::KeyFile));
-    } else {
-      shared_ptr<PSOBBEncryption::KeyFile> k(new PSOBBEncryption::KeyFile());
-      memcpy(k.get(), contents.data(), sizeof(PSOBBEncryption::KeyFile));
-      s->bb_private_keys.emplace_back(k);
-      log(INFO, "Loaded Blue Burst key file: %s", filename.c_str());
-    }
+    s->bb_private_keys.emplace_back(new PSOBBEncryption::KeyFile(
+        load_object_file<PSOBBEncryption::KeyFile>("system/blueburst/keys/" + filename)));
+    log(INFO, "Loaded Blue Burst key file: %s", filename.c_str());
   }
   log(INFO, "%zu Blue Burst key file(s) loaded", s->bb_private_keys.size());
 
@@ -283,6 +188,8 @@ enum class Behavior {
   RUN_SERVER = 0,
   DECRYPT_DATA,
   ENCRYPT_DATA,
+  DECODE_QUEST_FILE,
+  DECODE_SJIS,
 };
 
 enum class EncryptionType {
@@ -291,9 +198,17 @@ enum class EncryptionType {
   BB,
 };
 
+enum class QuestFileFormat {
+  GCI = 0,
+  DLQ,
+  QST,
+};
+
 int main(int argc, char** argv) {
   Behavior behavior = Behavior::RUN_SERVER;
   EncryptionType crypt_type = EncryptionType::PC;
+  QuestFileFormat quest_file_type = QuestFileFormat::GCI;
+  string quest_filename;
   string seed;
   string key_file_name;
   bool parse_data = false;
@@ -302,6 +217,20 @@ int main(int argc, char** argv) {
       behavior = Behavior::DECRYPT_DATA;
     } else if (!strcmp(argv[x], "--encrypt-data")) {
       behavior = Behavior::ENCRYPT_DATA;
+    } else if (!strcmp(argv[x], "--decode-sjis")) {
+      behavior = Behavior::DECODE_SJIS;
+    } else if (!strncmp(argv[x], "--decode-gci=", 13)) {
+      behavior = Behavior::DECODE_QUEST_FILE;
+      quest_file_type = QuestFileFormat::GCI;
+      quest_filename = &argv[x][13];
+    } else if (!strncmp(argv[x], "--decode-dlq=", 13)) {
+      behavior = Behavior::DECODE_QUEST_FILE;
+      quest_file_type = QuestFileFormat::DLQ;
+      quest_filename = &argv[x][13];
+    } else if (!strncmp(argv[x], "--decode-qst=", 13)) {
+      behavior = Behavior::DECODE_QUEST_FILE;
+      quest_file_type = QuestFileFormat::QST;
+      quest_filename = &argv[x][13];
     } else if (!strcmp(argv[x], "--pc")) {
       crypt_type = EncryptionType::PC;
     } else if (!strcmp(argv[x], "--gc")) {
@@ -319,7 +248,7 @@ int main(int argc, char** argv) {
     }
   }
 
-  if (behavior != Behavior::RUN_SERVER) {
+  if (behavior == Behavior::DECRYPT_DATA || behavior == Behavior::ENCRYPT_DATA) {
     shared_ptr<PSOEncryption> crypt;
     if (crypt_type == EncryptionType::PC) {
       crypt.reset(new PSOPCEncryption(stoul(seed, nullptr, 16)));
@@ -355,6 +284,30 @@ int main(int argc, char** argv) {
     fflush(stdout);
 
     return 0;
+
+  } else if (behavior == Behavior::DECODE_QUEST_FILE) {
+    if (quest_file_type == QuestFileFormat::GCI) {
+      save_file(quest_filename + ".dec", Quest::decode_gci(quest_filename));
+    } else if (quest_file_type == QuestFileFormat::DLQ) {
+      save_file(quest_filename + ".dec", Quest::decode_dlq(quest_filename));
+    } else if (quest_file_type == QuestFileFormat::QST) {
+      auto data = Quest::decode_qst(quest_filename);
+      save_file(quest_filename + ".bin", data.first);
+      save_file(quest_filename + ".dat", data.second);
+    } else {
+      throw logic_error("invalid quest file format");
+    }
+
+    return 0;
+
+  } else if (behavior == Behavior::DECODE_SJIS) {
+    string data = read_all(stdin);
+    if (parse_data) {
+      data = parse_data_string(data);
+    }
+    auto decoded = decode_sjis(data);
+    print_data(stderr, decoded.data(), decoded.size() * sizeof(decoded[0]));
+    return 0;
   }
 
   signal(SIGPIPE, SIG_IGN);
@@ -387,9 +340,21 @@ int main(int argc, char** argv) {
   log(INFO, "Loading level table");
   state->level_table.reset(new LevelTable("system/blueburst/PlyLevelTbl.prs", true));
 
+  log(INFO, "Collecting Episode 3 data");
+  state->ep3_data_index.reset(new Ep3DataIndex("system/ep3"));
+
   log(INFO, "Collecting quest metadata");
   state->quest_index.reset(new QuestIndex("system/quests"));
 
+  log(INFO, "Compiling client functions");
+  state->function_code_index.reset(new FunctionCodeIndex("system/ppc"));
+
+  log(INFO, "Loading DOL files");
+  state->dol_file_index.reset(new DOLFileIndex("system/dol"));
+
+  log(INFO, "Creating menus");
+  state->create_menus(config_json);
+
   shared_ptr<DNSServer> dns_server;
   if (state->dns_server_port) {
     log(INFO, "Starting DNS server");
@@ -400,8 +365,6 @@ int main(int argc, char** argv) {
     log(INFO, "DNS server is disabled");
   }
 
-  shared_ptr<Server> game_server;
-
   log(INFO, "Opening sockets");
   for (const auto& it : state->name_to_port_config) {
     const auto& pc = it.second;
@@ -431,22 +394,21 @@ int main(int argc, char** argv) {
         }
       }
     } else {
-      if (!game_server.get()) {
+      if (!state->game_server.get()) {
         log(INFO, "Starting game server");
-        game_server.reset(new Server(base, state));
+        state->game_server.reset(new Server(base, state));
       }
       string name = string_printf("%s (%s, %s) on port %hu",
           pc->name.c_str(), name_for_version(pc->version),
           name_for_server_behavior(pc->behavior), pc->port);
-      game_server->listen(name, "", pc->port, pc->version, pc->behavior);
+      state->game_server->listen(name, "", pc->port, pc->version, pc->behavior);
     }
   }
 
   shared_ptr<IPStackSimulator> ip_stack_simulator;
   if (!state->ip_stack_addresses.empty()) {
     log(INFO, "Starting IP stack simulator");
-    ip_stack_simulator.reset(new IPStackSimulator(
-        base, game_server, state));
+    ip_stack_simulator.reset(new IPStackSimulator(base, state));
     for (const auto& it : state->ip_stack_addresses) {
       auto netloc = parse_netloc(it);
       ip_stack_simulator->listen(netloc.first, netloc.second);

src/Menu.hh

@@ -6,21 +6,50 @@
 
 
 
-#define MAIN_MENU_ID                     0x11000011
-#define INFORMATION_MENU_ID              0x22000022
-#define LOBBY_MENU_ID                    0x33000033
-#define GAME_MENU_ID                     0x44000044
-#define QUEST_MENU_ID                    0x55000055
-#define QUEST_FILTER_MENU_ID             0x66000066
-#define PROXY_DESTINATIONS_MENU_ID       0x77000077
+// Note: These aren't enums because neither enum nor enum class does what we
+// want. Specifically, we need GO_BACK to be valid in multiple enums (and enums
+// aren't namespaced unless they're enum classes), so we can't use enums. But we
+// also want to be able to use non-enum values in switch statements without
+// casting values all over the place, so we can't use enum classes either.
 
-#define MAIN_MENU_GO_TO_LOBBY            0x11AAAA11
-#define MAIN_MENU_INFORMATION            0x11BBBB11
-#define MAIN_MENU_DOWNLOAD_QUESTS        0x11CCCC11
-#define MAIN_MENU_PROXY_DESTINATIONS     0x11DDDD11
-#define MAIN_MENU_DISCONNECT             0x11EEEE11
-#define INFORMATION_MENU_GO_BACK         0x22FFFF22
-#define PROXY_DESTINATIONS_MENU_GO_BACK  0x77FFFF77
+namespace MenuID {
+  constexpr uint32_t MAIN               = 0x11000011;
+  constexpr uint32_t INFORMATION        = 0x22000022;
+  constexpr uint32_t LOBBY              = 0x33000033;
+  constexpr uint32_t GAME               = 0x44000044;
+  constexpr uint32_t QUEST              = 0x55000055;
+  constexpr uint32_t QUEST_FILTER       = 0x66000066;
+  constexpr uint32_t PROXY_DESTINATIONS = 0x77000077;
+  constexpr uint32_t PROGRAMS           = 0x88000088;
+  constexpr uint32_t PATCHES            = 0x99000099;
+}
+
+namespace MainMenuItemID {
+  constexpr uint32_t GO_TO_LOBBY        = 0x11222211;
+  constexpr uint32_t INFORMATION        = 0x11333311;
+  constexpr uint32_t DOWNLOAD_QUESTS    = 0x11444411;
+  constexpr uint32_t PROXY_DESTINATIONS = 0x11555511;
+  constexpr uint32_t PATCHES            = 0x11666611;
+  constexpr uint32_t PROGRAMS           = 0x11777711;
+  constexpr uint32_t DISCONNECT         = 0x11888811;
+  constexpr uint32_t CLEAR_LICENSE      = 0x11999911;
+}
+
+namespace InformationMenuItemID {
+  constexpr uint32_t GO_BACK = 0x22FFFF22;
+};
+
+namespace ProxyDestinationsMenuItemID {
+  constexpr uint32_t GO_BACK = 0x77FFFF77;
+};
+
+namespace ProgramsMenuItemID {
+  constexpr uint32_t GO_BACK = 0x88FFFF88;
+};
+
+namespace PatchesMenuItemID {
+  constexpr uint32_t GO_BACK = 0x99FFFF99;
+};
 
 
 
@@ -35,6 +64,8 @@ struct MenuItem {
     GC_ONLY = INVISIBLE_ON_DC | INVISIBLE_ON_PC | INVISIBLE_ON_BB,
     BB_ONLY = INVISIBLE_ON_DC | INVISIBLE_ON_PC | INVISIBLE_ON_GC,
     REQUIRES_MESSAGE_BOXES = 0x10,
+    REQUIRES_SEND_FUNCTION_CALL = 0x20,
+    REQUIRES_SAVE_DISABLED = 0x40,
   };
 
   uint32_t item_id;

src/PSOEncryption.cc

@@ -176,48 +176,37 @@ void PSOGCEncryption::encrypt(void* vdata, size_t size, bool advance) {
 
 
 void PSOBBEncryption::decrypt(void* vdata, size_t size, bool advance) {
-  if (this->subtype != Subtype::JSD1) {
+  if (this->state.subtype == Subtype::TFS1) {
     if (size & 7) {
       throw invalid_argument("size must be a multiple of 8");
     }
-    size_t num_dwords = size >> 2;
-    le_uint32_t* dwords = reinterpret_cast<le_uint32_t*>(vdata);
-    uint32_t edx, ebx, ebp, esi, edi;
 
-    edx = 0;
-    while (edx < num_dwords) {
-      ebx = dwords[edx];
-      ebx = ebx ^ this->stream[5];
-      ebp = ((this->stream[(ebx >> 0x18) + 0x12] +
-              this->stream[((ebx >> 0x10) & 0xFF) + 0x112]) ^
-             this->stream[((ebx >> 0x8) & 0xFF) + 0x212]) +
-            this->stream[(ebx & 0xFF) + 0x312];
-      ebp = ebp ^ this->stream[4];
-      ebp ^= dwords[edx + 1];
-      edi = ((this->stream[(ebp >> 0x18) + 0x12] +
-              this->stream[((ebp >> 0x10) & 0xFF) + 0x112]) ^
-             this->stream[((ebp >> 0x8) & 0xFF) + 0x212]) +
-            this->stream[(ebp & 0xFF) + 0x312];
-      edi = edi ^ this->stream[3];
-      ebx = ebx ^ edi;
-      esi = ((this->stream[(ebx >> 0x18) + 0x12] +
-              this->stream[((ebx >> 0x10) & 0xFF) + 0x112]) ^
-             this->stream[((ebx >> 0x8) & 0xFF) + 0x212]) +
-            this->stream[(ebx & 0xFF) + 0x312];
-      ebp = ebp ^ esi ^ this->stream[2];
-      edi = ((this->stream[(ebp >> 0x18) + 0x12] +
-              this->stream[((ebp >> 0x10) & 0xFF) + 0x112]) ^
-             this->stream[((ebp >> 0x8) & 0xFF) + 0x212]) +
-            this->stream[(ebp & 0xFF) + 0x312];
-      edi = edi ^ this->stream[1];
-      ebp = ebp ^ this->stream[0];
-      ebx = ebx ^ edi;
-      dwords[edx] = ebp;
-      dwords[edx + 1] = ebx;
-      edx += 2;
+    le_uint32_t* dwords = reinterpret_cast<le_uint32_t*>(vdata);
+    for (size_t x = 0; x < (size >> 2); x += 2) {
+      for (size_t y = 4; y > 0; y -= 2) {
+        dwords[x] = dwords[x] ^ this->state.initial_keys.as32[y + 1];
+        dwords[x + 1] ^= ((this->state.private_keys.as32[dwords[x] >> 24] +
+                             this->state.private_keys.as32[((dwords[x] >> 16) & 0xFF) + 0x100]) ^
+                            this->state.private_keys.as32[((dwords[x] >> 8) & 0xFF) + 0x200]) +
+                           this->state.private_keys.as32[(dwords[x] & 0xFF) + 0x300];
+        dwords[x + 1] ^= this->state.initial_keys.as32[y];
+        dwords[x] ^= ((this->state.private_keys.as32[dwords[x + 1] >> 24] +
+                         this->state.private_keys.as32[((dwords[x + 1] >> 16) & 0xFF) + 0x100]) ^
+                        this->state.private_keys.as32[((dwords[x + 1] >> 8) & 0xFF) + 0x200]) +
+                       this->state.private_keys.as32[(dwords[x + 1] & 0xFF) + 0x300];
+      }
+      dwords[x] ^= this->state.initial_keys.as32[1];
+      dwords[x + 1] ^= this->state.initial_keys.as32[0];
+
+      uint32_t a = dwords[x];
+      dwords[x] = dwords[x + 1];
+      dwords[x + 1] = a;
     }
 
-  } else { // subtype == Subtype::JSD1
+  } else if (this->state.subtype == Subtype::JSD1) {
+    if (size & 1) {
+      throw invalid_argument("size must be a multiple of 2");
+    }
     if (!advance && (size > 0x100)) {
       throw logic_error("JSD1 can only peek-decrypt up to 0x100 bytes");
     }
@@ -228,22 +217,115 @@ void PSOBBEncryption::decrypt(void* vdata, size_t size, bool advance) {
       bytes[z] = (a & 0x55) | (b & 0xAA);
       bytes[z + 1] = (a & 0xAA) | (b & 0x55);
     }
-    uint8_t* stream_bytes = reinterpret_cast<uint8_t*>(this->stream.data());
     for (size_t z = 0; z < size; z++) {
-      bytes[z] ^= stream_bytes[this->jsd1_stream_offset];
+      bytes[z] ^= this->state.private_keys.as8[this->state.initial_keys.jsd1_stream_offset];
       if (advance) {
-        stream_bytes[this->jsd1_stream_offset] -= bytes[z];
+        this->state.private_keys.as8[this->state.initial_keys.jsd1_stream_offset] -= bytes[z];
       }
-      this->jsd1_stream_offset++;
+      this->state.initial_keys.jsd1_stream_offset++;
     }
     if (!advance) {
-      this->jsd1_stream_offset -= size;
+      this->state.initial_keys.jsd1_stream_offset -= size;
+    }
+
+  } else { // STANDARD or MOCB1
+    if (size & 7) {
+      throw invalid_argument("size must be a multiple of 8");
+    }
+    size_t num_dwords = size >> 2;
+    le_uint32_t* dwords = reinterpret_cast<le_uint32_t*>(vdata);
+    uint32_t edx, ebx, ebp, esi, edi;
+
+    edx = 0;
+    while (edx < num_dwords) {
+      ebx = dwords[edx];
+      ebx = ebx ^ this->state.initial_keys.as32[5];
+      ebp = ((this->state.private_keys.as32[(ebx >> 0x18)] +
+              this->state.private_keys.as32[((ebx >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebx >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebx & 0xFF) + 0x300];
+      ebp = ebp ^ this->state.initial_keys.as32[4];
+      ebp ^= dwords[edx + 1];
+      edi = ((this->state.private_keys.as32[(ebp >> 0x18)] +
+              this->state.private_keys.as32[((ebp >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebp >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebp & 0xFF) + 0x300];
+      edi = edi ^ this->state.initial_keys.as32[3];
+      ebx = ebx ^ edi;
+      esi = ((this->state.private_keys.as32[(ebx >> 0x18)] +
+              this->state.private_keys.as32[((ebx >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebx >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebx & 0xFF) + 0x300];
+      ebp = ebp ^ esi ^ this->state.initial_keys.as32[2];
+      edi = ((this->state.private_keys.as32[(ebp >> 0x18)] +
+              this->state.private_keys.as32[((ebp >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebp >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebp & 0xFF) + 0x300];
+      edi = edi ^ this->state.initial_keys.as32[1];
+      ebp = ebp ^ this->state.initial_keys.as32[0];
+      ebx = ebx ^ edi;
+      dwords[edx] = ebp;
+      dwords[edx + 1] = ebx;
+      edx += 2;
     }
   }
 }
 
 void PSOBBEncryption::encrypt(void* vdata, size_t size, bool advance) {
-  if (this->subtype != Subtype::JSD1) {
+  if (this->state.subtype == Subtype::TFS1) {
+    if (size & 7) {
+      throw invalid_argument("size must be a multiple of 8");
+    }
+
+    le_uint32_t* dwords = reinterpret_cast<le_uint32_t*>(vdata);
+    for (size_t x = 0; x < (size >> 2); x += 2) {
+      for (size_t y = 0; y < 4; y += 2) {
+        dwords[x] ^= this->state.initial_keys.as32[y];
+        dwords[x + 1] ^= ((this->state.private_keys.as32[dwords[x] >> 24] +
+                         this->state.private_keys.as32[((dwords[x] >> 16) & 0xFF) + 0x100]) ^
+                        this->state.private_keys.as32[((dwords[x] >> 8) & 0xFF) + 0x200]) +
+                       this->state.private_keys.as32[(dwords[x] & 0xFF) + 0x300];
+        dwords[x + 1] ^= this->state.initial_keys.as32[y + 1];
+        dwords[x] ^= ((this->state.private_keys.as32[dwords[x + 1] >> 24] +
+                     this->state.private_keys.as32[(dwords[x + 1] >> 16 & 0xFF) + 0x100]) ^
+                    this->state.private_keys.as32[(dwords[x + 1] >> 8 & 0xFF) + 0x200]) +
+                   this->state.private_keys.as32[(dwords[x + 1] & 0xFF) + 0x300];
+      }
+      dwords[x] ^= this->state.initial_keys.as32[4];
+      dwords[x + 1] ^= this->state.initial_keys.as32[5];
+
+      uint32_t a = dwords[x];
+      dwords[x] = dwords[x + 1];
+      dwords[x + 1] = a;
+    }
+
+  } else if (this->state.subtype == Subtype::JSD1) {
+    if (size & 1) {
+      throw invalid_argument("size must be a multiple of 2");
+    }
+    if (!advance && (size > 0x100)) {
+      throw logic_error("JSD1 can only peek-encrypt up to 0x100 bytes");
+    }
+    uint8_t* bytes = reinterpret_cast<uint8_t*>(vdata);
+    for (size_t z = 0; z < size; z++) {
+      uint8_t v = bytes[z];
+      bytes[z] = v ^ this->state.private_keys.as8[this->state.initial_keys.jsd1_stream_offset];
+      if (advance) {
+        this->state.private_keys.as8[this->state.initial_keys.jsd1_stream_offset] -= v;
+      }
+      this->state.initial_keys.jsd1_stream_offset++;
+    }
+    if (!advance) {
+      this->state.initial_keys.jsd1_stream_offset -= size;
+    }
+    for (size_t z = 0; z < size; z += 2) {
+      uint8_t a = bytes[z];
+      uint8_t b = bytes[z + 1];
+      bytes[z] = (a & 0x55) | (b & 0xAA);
+      bytes[z + 1] = (a & 0xAA) | (b & 0x55);
+    }
+
+  } else { // STANDARD or MOCB1
     if (size & 7) {
       throw invalid_argument("size must be a multiple of 8");
     }
@@ -254,58 +336,63 @@ void PSOBBEncryption::encrypt(void* vdata, size_t size, bool advance) {
 
     edx = 0;
     while (edx < num_dwords) {
-      ebx = data[edx] ^ this->stream[0];
-      ebp = ((this->stream[(ebx >> 0x18) + 0x12] + this->stream[((ebx >> 0x10) & 0xFF) + 0x112])
-          ^ this->stream[((ebx >> 0x8) & 0xFF) + 0x212]) + this->stream[(ebx & 0xFF) + 0x312];
-      ebp = ebp ^ this->stream[1];
+      ebx = data[edx] ^ this->state.initial_keys.as32[0];
+      ebp = ((this->state.private_keys.as32[(ebx >> 0x18)] +
+              this->state.private_keys.as32[((ebx >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebx >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebx & 0xFF) + 0x300];
+      ebp = ebp ^ this->state.initial_keys.as32[1];
       ebp ^= data[edx + 1];
-      edi = ((this->stream[(ebp >> 0x18) + 0x12] + this->stream[((ebp >> 0x10) & 0xFF) + 0x112])
-          ^ this->stream[((ebp >> 0x8) & 0xFF) + 0x212]) + this->stream[(ebp & 0xFF) + 0x312];
-      edi = edi ^ this->stream[2];
+      edi = ((this->state.private_keys.as32[(ebp >> 0x18)] +
+              this->state.private_keys.as32[((ebp >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebp >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebp & 0xFF) + 0x300];
+      edi = edi ^ this->state.initial_keys.as32[2];
       ebx = ebx ^ edi;
-      esi = ((this->stream[(ebx >> 0x18) + 0x12] + this->stream[((ebx >> 0x10) & 0xFF) + 0x112])
-          ^ this->stream[((ebx >> 0x8) & 0xFF) + 0x212]) + this->stream[(ebx & 0xFF) + 0x312];
-      ebp = ebp ^ esi ^ this->stream[3];
-      edi = ((this->stream[(ebp >> 0x18) + 0x12] + this->stream[((ebp >> 0x10) & 0xFF) + 0x112])
-          ^ this->stream[((ebp >> 0x8) & 0xFF) + 0x212]) + this->stream[(ebp & 0xFF) + 0x312];
-      edi = edi ^ this->stream[4];
-      ebp = ebp ^ this->stream[5];
+      esi = ((this->state.private_keys.as32[(ebx >> 0x18)] +
+              this->state.private_keys.as32[((ebx >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebx >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebx & 0xFF) + 0x300];
+      ebp = ebp ^ esi ^ this->state.initial_keys.as32[3];
+      edi = ((this->state.private_keys.as32[(ebp >> 0x18)] +
+              this->state.private_keys.as32[((ebp >> 0x10) & 0xFF) + 0x100]) ^
+             this->state.private_keys.as32[((ebp >> 0x8) & 0xFF) + 0x200]) +
+            this->state.private_keys.as32[(ebp & 0xFF) + 0x300];
+      edi = edi ^ this->state.initial_keys.as32[4];
+      ebp = ebp ^ this->state.initial_keys.as32[5];
       ebx = ebx ^ edi;
       data[edx] = ebp;
       data[edx + 1] = ebx;
       edx += 2;
     }
-
-  } else { // subtype == Subtype::JSD1
-    if (!advance && (size > 0x100)) {
-      throw logic_error("JSD1 can only peek-encrypt up to 0x100 bytes");
-    }
-    uint8_t* bytes = reinterpret_cast<uint8_t*>(vdata);
-    uint8_t* stream_bytes = reinterpret_cast<uint8_t*>(this->stream.data());
-    for (size_t z = 0; z < size; z++) {
-      uint8_t v = bytes[z];
-      bytes[z] = v ^ stream_bytes[this->jsd1_stream_offset];
-      if (advance) {
-        stream_bytes[this->jsd1_stream_offset] -= v;
-      }
-      this->jsd1_stream_offset++;
-    }
-    if (!advance) {
-      this->jsd1_stream_offset -= size;
-    }
-    for (size_t z = 0; z < size; z += 2) {
-      uint8_t a = bytes[z];
-      uint8_t b = bytes[z + 1];
-      bytes[z] = (a & 0x55) | (b & 0xAA);
-      bytes[z + 1] = (a & 0xAA) | (b & 0x55);
-    }
   }
 }
 
 PSOBBEncryption::PSOBBEncryption(
     const KeyFile& key, const void* original_seed, size_t seed_size)
-  : subtype(key.subtype), jsd1_stream_offset(0) {
+  : state(key) {
+  this->apply_seed(original_seed, seed_size);
+}
 
+void PSOBBEncryption::tfs1_scramble(uint32_t* out1, uint32_t* out2) const {
+  uint32_t a = *out1;
+  uint32_t b = *out2;
+  for (size_t x = 0; x < 0x10; x += 2) {
+    a ^= this->state.initial_keys.as32[x];
+    b ^= (((this->state.private_keys.as32[a >> 24] +
+            this->state.private_keys.as32[((a >> 16) & 0xFF) + 0x100]) ^
+           this->state.private_keys.as32[((a >> 8) & 0xFF) + 0x200]) +
+          this->state.private_keys.as32[(a & 0xFF) + 0x300]) ^ this->state.initial_keys.as32[x + 1];
+    a ^= ((this->state.private_keys.as32[b >> 24] +
+           this->state.private_keys.as32[((b >> 16) & 0xFF) + 0x100]) ^
+          this->state.private_keys.as32[((b >> 8) & 0xFF) + 0x200]) +
+         this->state.private_keys.as32[(b & 0xFF) + 0x300];
+  }
+  *out1 = this->state.initial_keys.as32[0x11] ^ b;
+  *out2 = this->state.initial_keys.as32[0x10] ^ a;
+}
+
+void PSOBBEncryption::apply_seed(const void* original_seed, size_t seed_size) {
   // Note: This part is done in the 03 command handler in the BB client, and
   // isn't actually part of the encryption library. (Why did they do this?)
   string seed;
@@ -317,25 +404,55 @@ PSOBBEncryption::PSOBBEncryption(
     seed.push_back(original_seed_data[x + 2] ^ 0x18);
   }
 
-  if (this->subtype != Subtype::JSD1) {
+  if (this->state.subtype == Subtype::TFS1) {
+    for (size_t x = 0; x < 0x12; x++) {
+      uint32_t a = this->state.initial_keys.as32[x] & 0xFFFF;
+      this->state.initial_keys.as32[x] = ((a << 0x10) ^ (this->state.initial_keys.as32[x] & 0xFFFF0000)) + a;
+    };
+
+    const uint8_t* useed = reinterpret_cast<const uint8_t*>(seed.data());
+    for (size_t x = 0; x < 0x48; x += 4) {
+      uint32_t seed_data =
+          (useed[x % seed_size] << 24) |
+          (useed[(x + 1) % seed_size] << 16) |
+          (useed[(x + 2) % seed_size] << 8) |
+          useed[(x + 3) % seed_size];
+      this->state.initial_keys.as32[x >> 2] ^= seed_data;
+    }
+
+    uint32_t a = 0, b = 0;
+    for (size_t x = 0; x < 0x12; x += 2) {
+      this->tfs1_scramble(&a, &b);
+      this->state.initial_keys.as32[x] = a;
+      this->state.initial_keys.as32[x + 1] = b;
+    }
+
+    for (size_t x = 0; x < 0x400; x += 2) {
+      this->tfs1_scramble(&a, &b);
+      this->state.private_keys.as32[x] = a;
+      this->state.private_keys.as32[x + 1] = b;
+    }
+
+  } else if (this->state.subtype == Subtype::JSD1) {
+    size_t seed_offset = 0;
+    for (size_t z = 0; z < 0x100; z++) {
+      this->state.private_keys.as8[z] = (z + seed[seed_offset]) ^ (static_cast<uint8_t>(seed[seed_offset]) >> 1);
+      seed_offset = (seed_offset + 1) % seed.size();
+    }
+
+  } else { // STANDARD or MOCB1 (they share most of their logic)
     if (seed_size % 3) {
       throw invalid_argument("seed size must be divisible by 3");
     }
 
-    this->stream.resize(BB_STREAM_LENGTH, 0);
-
-    if (key.subtype == Subtype::MOCB1) {
+    if (this->state.subtype == Subtype::MOCB1) {
       for (size_t x = 0; x < 0x12; x++) {
-        uint8_t a = key.initial_keys[4 * x + 0];
-        uint8_t b = key.initial_keys[4 * x + 1];
-        uint8_t c = key.initial_keys[4 * x + 2];
-        uint8_t d = key.initial_keys[4 * x + 3];
-        this->stream[x] = ((a ^ d) << 24) | ((b ^ c) << 16) | (a << 8) | b;
+        uint8_t a = this->state.initial_keys.as8[4 * x + 0];
+        uint8_t b = this->state.initial_keys.as8[4 * x + 1];
+        uint8_t c = this->state.initial_keys.as8[4 * x + 2];
+        uint8_t d = this->state.initial_keys.as8[4 * x + 3];
+        this->state.initial_keys.as32[x] = ((a ^ d) << 24) | ((b ^ c) << 16) | (a << 8) | b;
       }
-      memcpy(this->stream.data() + 0x12, &key.private_keys, sizeof(key.private_keys));
-
-    } else {
-      memcpy(this->stream.data(), &key, sizeof(key));
     }
 
     // This block was formerly postprocess_initial_stream
@@ -362,7 +479,7 @@ PSOBBEncryption::PSOBBEncryption(
         ebp = ebp | eax;
         eax = ecx;
         edx = eax % seed.size();
-        this->stream[ebx] ^= ebp;
+        this->state.initial_keys.as32[ebx] ^= ebp;
         ecx = edx;
         ebx++;
       }
@@ -375,73 +492,73 @@ PSOBBEncryption::PSOBBEncryption(
       edx = 0x48;
 
       while (edi < edx) {
-        esi = esi ^ this->stream[0];
+        esi = esi ^ this->state.initial_keys.as32[0];
         eax = esi >> 0x18;
         ebx = (esi >> 0x10) & 0xFF;
-        eax = this->stream[eax + 0x12] + this->stream[ebx + 0x112];
+        eax = this->state.private_keys.as32[eax] + this->state.private_keys.as32[ebx + 0x100];
         ebx = (esi >> 8) & 0xFF;
-        eax = eax ^ this->stream[ebx + 0x212];
+        eax = eax ^ this->state.private_keys.as32[ebx + 0x200];
         ebx = esi & 0xFF;
-        eax = eax + this->stream[ebx + 0x312];
+        eax = eax + this->state.private_keys.as32[ebx + 0x300];
 
-        eax = eax ^ this->stream[1];
+        eax = eax ^ this->state.initial_keys.as32[1];
         ecx = ecx ^ eax;
         ebx = ecx >> 0x18;
         eax = (ecx >> 0x10) & 0xFF;
-        ebx = this->stream[ebx + 0x12] + this->stream[eax + 0x112];
+        ebx = this->state.private_keys.as32[ebx] + this->state.private_keys.as32[eax + 0x100];
         eax = (ecx >> 8) & 0xFF;
-        ebx = ebx ^ this->stream[eax + 0x212];
+        ebx = ebx ^ this->state.private_keys.as32[eax + 0x200];
         eax = ecx & 0xFF;
-        ebx = ebx + this->stream[eax + 0x312];
+        ebx = ebx + this->state.private_keys.as32[eax + 0x300];
 
         for (x = 0; x <= 5; x++) {
-          ebx = ebx ^ this->stream[(x * 2) + 2];
+          ebx = ebx ^ this->state.initial_keys.as32[(x * 2) + 2];
           esi = esi ^ ebx;
           ebx = esi >> 0x18;
           eax = (esi >> 0x10) & 0xFF;
-          ebx = this->stream[ebx + 0x12] + this->stream[eax + 0x112];
+          ebx = this->state.private_keys.as32[ebx] + this->state.private_keys.as32[eax + 0x100];
           eax = (esi >> 8) & 0xFF;
-          ebx = ebx ^ this->stream[eax + 0x212];
+          ebx = ebx ^ this->state.private_keys.as32[eax + 0x200];
           eax = esi & 0xFF;
-          ebx = ebx + this->stream[eax + 0x312];
+          ebx = ebx + this->state.private_keys.as32[eax + 0x300];
 
-          ebx = ebx ^ this->stream[(x * 2) + 3];
+          ebx = ebx ^ this->state.initial_keys.as32[(x * 2) + 3];
           ecx = ecx ^ ebx;
           ebx = ecx >> 0x18;
           eax = (ecx >> 0x10) & 0xFF;
-          ebx = this->stream[ebx + 0x12] + this->stream[eax + 0x112];
+          ebx = this->state.private_keys.as32[ebx] + this->state.private_keys.as32[eax + 0x100];
           eax = (ecx >> 8) & 0xFF;
-          ebx = ebx ^ this->stream[eax + 0x212];
+          ebx = ebx ^ this->state.private_keys.as32[eax + 0x200];
           eax = ecx & 0xFF;
-          ebx = ebx + this->stream[eax + 0x312];
+          ebx = ebx + this->state.private_keys.as32[eax + 0x300];
         }
 
-        ebx = ebx ^ this->stream[14];
+        ebx = ebx ^ this->state.initial_keys.as32[14];
         esi = esi ^ ebx;
         eax = esi >> 0x18;
         ebx = (esi >> 0x10) & 0xFF;
-        eax = this->stream[eax + 0x12] + this->stream[ebx + 0x112];
+        eax = this->state.private_keys.as32[eax] + this->state.private_keys.as32[ebx + 0x100];
         ebx = (esi >> 8) & 0xFF;
-        eax = eax ^ this->stream[ebx + 0x212];
+        eax = eax ^ this->state.private_keys.as32[ebx + 0x200];
         ebx = esi & 0xFF;
-        eax = eax + this->stream[ebx + 0x312];
+        eax = eax + this->state.private_keys.as32[ebx + 0x300];
 
-        eax = eax ^ this->stream[15];
+        eax = eax ^ this->state.initial_keys.as32[15];
         eax = ecx ^ eax;
         ecx = eax >> 0x18;
         ebx = (eax >> 0x10) & 0xFF;
-        ecx = this->stream[ecx + 0x12] + this->stream[ebx + 0x112];
+        ecx = this->state.private_keys.as32[ecx] + this->state.private_keys.as32[ebx + 0x100];
         ebx = (eax >> 8) & 0xFF;
-        ecx = ecx ^ this->stream[ebx + 0x212];
+        ecx = ecx ^ this->state.private_keys.as32[ebx + 0x200];
         ebx = eax & 0xFF;
-        ecx = ecx + this->stream[ebx + 0x312];
+        ecx = ecx + this->state.private_keys.as32[ebx + 0x300];
 
-        ecx = ecx ^ this->stream[16];
+        ecx = ecx ^ this->state.initial_keys.as32[16];
         ecx = ecx ^ esi;
-        esi = this->stream[17];
+        esi = this->state.initial_keys.as32[17];
         esi = esi ^ eax;
-        this->stream[(edi / 4)] = esi;
-        this->stream[(edi / 4)+1] = ecx;
+        this->state.initial_keys.as32[(edi / 4)] = esi;
+        this->state.initial_keys.as32[(edi / 4)+1] = ecx;
         edi = edi + 8;
       }
 
@@ -449,91 +566,82 @@ PSOBBEncryption::PSOBBEncryption(
       edx = 0;
       ou = 0;
       while (ou < 0x1000) {
-        edi = 0x48;
-        edx = 0x448;
+        edi = 0;
+        edx = 0x400;
 
         while (edi < edx) {
-          esi = esi ^ this->stream[0];
+          esi = esi ^ this->state.initial_keys.as32[0];
           eax = esi >> 0x18;
           ebx = (esi >> 0x10) & 0xFF;
-          eax = this->stream[eax + 0x12] + this->stream[ebx + 0x112];
+          eax = this->state.private_keys.as32[eax] + this->state.private_keys.as32[ebx + 0x100];
           ebx = (esi >> 8) & 0xFF;
-          eax = eax ^ this->stream[ebx + 0x212];
+          eax = eax ^ this->state.private_keys.as32[ebx + 0x200];
           ebx = esi & 0xFF;
-          eax = eax + this->stream[ebx + 0x312];
+          eax = eax + this->state.private_keys.as32[ebx + 0x300];
 
-          eax = eax ^ this->stream[1];
+          eax = eax ^ this->state.initial_keys.as32[1];
           ecx = ecx ^ eax;
           ebx = ecx >> 0x18;
           eax = (ecx >> 0x10) & 0xFF;
-          ebx = this->stream[ebx + 0x12] + this->stream[eax + 0x112];
+          ebx = this->state.private_keys.as32[ebx] + this->state.private_keys.as32[eax + 0x100];
           eax = (ecx >> 8) & 0xFF;
-          ebx = ebx ^ this->stream[eax + 0x212];
+          ebx = ebx ^ this->state.private_keys.as32[eax + 0x200];
           eax = ecx & 0xFF;
-          ebx = ebx + this->stream[eax + 0x312];
+          ebx = ebx + this->state.private_keys.as32[eax + 0x300];
 
           for (x = 0; x <= 5; x++) {
-            ebx = ebx ^ this->stream[(x * 2) + 2];
+            ebx = ebx ^ this->state.initial_keys.as32[(x * 2) + 2];
             esi = esi ^ ebx;
             ebx = esi >> 0x18;
             eax = (esi >> 0x10) & 0xFF;
-            ebx = this->stream[ebx + 0x12] + this->stream[eax + 0x112];
+            ebx = this->state.private_keys.as32[ebx] + this->state.private_keys.as32[eax + 0x100];
             eax = (esi >> 8) & 0xFF;
-            ebx = ebx ^ this->stream[eax + 0x212];
+            ebx = ebx ^ this->state.private_keys.as32[eax + 0x200];
             eax = esi & 0xFF;
-            ebx = ebx + this->stream[eax + 0x312];
+            ebx = ebx + this->state.private_keys.as32[eax + 0x300];
 
-            ebx = ebx ^ this->stream[(x * 2) + 3];
+            ebx = ebx ^ this->state.initial_keys.as32[(x * 2) + 3];
             ecx = ecx ^ ebx;
             ebx = ecx >> 0x18;
             eax = (ecx >> 0x10) & 0xFF;
-            ebx = this->stream[ebx + 0x12] + this->stream[eax + 0x112];
+            ebx = this->state.private_keys.as32[ebx] + this->state.private_keys.as32[eax + 0x100];
             eax = (ecx >> 8) & 0xFF;
-            ebx = ebx ^ this->stream[eax + 0x212];
+            ebx = ebx ^ this->state.private_keys.as32[eax + 0x200];
             eax = ecx & 0xFF;
-            ebx = ebx + this->stream[eax + 0x312];
+            ebx = ebx + this->state.private_keys.as32[eax + 0x300];
           }
 
-          ebx = ebx ^ this->stream[14];
+          ebx = ebx ^ this->state.initial_keys.as32[14];
           esi = esi ^ ebx;
           eax = esi >> 0x18;
           ebx = (esi >> 0x10) & 0xFF;
-          eax = this->stream[eax + 0x12] + this->stream[ebx + 0x112];
+          eax = this->state.private_keys.as32[eax] + this->state.private_keys.as32[ebx + 0x100];
           ebx = (esi >> 8) & 0xFF;
-          eax = eax ^ this->stream[ebx + 0x212];
+          eax = eax ^ this->state.private_keys.as32[ebx + 0x200];
           ebx = esi & 0xFF;
-          eax = eax + this->stream[ebx + 0x312];
+          eax = eax + this->state.private_keys.as32[ebx + 0x300];
 
-          eax = eax ^ this->stream[15];
+          eax = eax ^ this->state.initial_keys.as32[15];
           eax = ecx ^ eax;
           ecx = eax >> 0x18;
           ebx = (eax >> 0x10) & 0xFF;
-          ecx = this->stream[ecx + 0x12] + this->stream[ebx + 0x112];
+          ecx = this->state.private_keys.as32[ecx] + this->state.private_keys.as32[ebx + 0x100];
           ebx = (eax >> 8) & 0xFF;
-          ecx = ecx ^ this->stream[ebx + 0x212];
+          ecx = ecx ^ this->state.private_keys.as32[ebx + 0x200];
           ebx = eax & 0xFF;
-          ecx = ecx + this->stream[ebx + 0x312];
+          ecx = ecx + this->state.private_keys.as32[ebx + 0x300];
 
-          ecx = ecx ^ this->stream[16];
+          ecx = ecx ^ this->state.initial_keys.as32[16];
           ecx = ecx ^ esi;
-          esi = this->stream[17];
+          esi = this->state.initial_keys.as32[17];
           esi = esi ^ eax;
-          this->stream[(ou / 4) + (edi / 4)] = esi;
-          this->stream[(ou / 4) + (edi / 4) + 1] = ecx;
+          this->state.private_keys.as32[(ou / 4) + (edi / 4)] = esi;
+          this->state.private_keys.as32[(ou / 4) + (edi / 4) + 1] = ecx;
           edi = edi + 8;
         }
         ou = ou + 0x400;
       }
     }
-
-  } else { // subtype == Subtype::JSD1
-    this->stream.resize(0x40);
-    uint8_t* stream_bytes = reinterpret_cast<uint8_t*>(this->stream.data());
-    size_t seed_offset = 0;
-    for (size_t z = 0; z < 0x100; z++) {
-      stream_bytes[z] = (z + seed[seed_offset]) ^ (static_cast<uint8_t>(seed[seed_offset]) >> 1);
-      seed_offset = (seed_offset + 1) % seed.size();
-    }
   }
 }
 

src/PSOEncryption.hh

@@ -6,6 +6,9 @@
 #include <memory>
 #include <string>
 #include <vector>
+#include <phosg/Encoding.hh>
+
+#include "Text.hh" // for parray
 
 
 
@@ -65,13 +68,27 @@ public:
     STANDARD = 0x00,
     MOCB1 = 0x01,
     JSD1 = 0x02,
+    TFS1 = 0x03,
   };
 
   struct KeyFile {
     // initial_keys are actually a stream of uint32_ts, but we treat them as
     // bytes for code simplicity
-    uint8_t initial_keys[0x12 * 4];
-    uint32_t private_keys[0x400];
+    union InitialKeys {
+      uint8_t jsd1_stream_offset;
+      parray<uint8_t, 0x48> as8;
+      parray<le_uint32_t, 0x12> as32;
+      InitialKeys() : as32() { }
+      InitialKeys(const InitialKeys& other) : as32(other.as32) { }
+    } __attribute__((packed));
+    union PrivateKeys {
+      parray<uint8_t, 0x1000> as8;
+      parray<le_uint32_t, 0x400> as32;
+      PrivateKeys() : as32() { }
+      PrivateKeys(const PrivateKeys& other) : as32(other.as32) { }
+    } __attribute__((packed));
+    InitialKeys initial_keys;
+    PrivateKeys private_keys;
     Subtype subtype;
   } __attribute__((packed));
 
@@ -81,9 +98,10 @@ public:
   virtual void decrypt(void* data, size_t size, bool advance = true);
 
 protected:
-  Subtype subtype;
-  std::vector<uint32_t> stream;
-  uint8_t jsd1_stream_offset;
+  KeyFile state;
+
+  void tfs1_scramble(uint32_t* out1, uint32_t* out2) const;
+  void apply_seed(const void* original_seed, size_t seed_size);
 };
 
 // The following classes provide support for multiple PSOBB private keys, and

src/PSOProtocol.cc

@@ -131,102 +131,6 @@ void PSOCommandHeader::set_flag(GameVersion version, uint32_t flag) {
 
 
 
-void for_each_received_command(
-    struct bufferevent* bev,
-    GameVersion version,
-    PSOEncryption* crypt,
-    function<void(uint16_t, uint16_t, string&)> fn) {
-  struct evbuffer* buf = bufferevent_get_input(bev);
-
-  size_t header_size = (version == GameVersion::BB) ? 8 : 4;
-  for (;;) {
-    PSOCommandHeader header;
-    if (evbuffer_copyout(buf, &header, header_size)
-        < static_cast<ssize_t>(header_size)) {
-      break;
-    }
-
-    if (crypt) {
-      crypt->decrypt(&header, header_size, false);
-    }
-
-    size_t command_logical_size = header.size(version);
-
-    // If encryption is enabled, BB pads commands to 8-byte boundaries, and this
-    // is not reflected in the size field. This logic does not occur if
-    // encryption is not yet enabled.
-    size_t command_physical_size = (crypt && (version == GameVersion::BB))
-        ? ((command_logical_size + header_size - 1) & ~(header_size - 1))
-        : command_logical_size;
-    if (evbuffer_get_length(buf) < command_physical_size) {
-      break;
-    }
-
-    // If we get here, then there is a full command in the buffer. Some
-    // encryption algorithms' advancement depends on the decrypted data, so we
-    // have to actually decrypt the header again (with advance=true) to keep
-    // them in a consistent state.
-
-    string header_data(header_size, '\0');
-    if (evbuffer_remove(buf, header_data.data(), header_data.size())
-        < static_cast<ssize_t>(header_data.size())) {
-      throw logic_error("enough bytes available, but could not remove them");
-    }
-
-    string command_data(command_physical_size - header_size, '\0');
-    if (evbuffer_remove(buf, command_data.data(), command_data.size())
-        < static_cast<ssize_t>(command_data.size())) {
-      throw logic_error("enough bytes available, but could not remove them");
-    }
-
-    if (crypt) {
-      crypt->decrypt(header_data.data(), header_data.size());
-      crypt->decrypt(command_data.data(), command_data.size());
-    }
-    command_data.resize(command_logical_size - header_size);
-
-    fn(header.command(version), header.flag(version), command_data);
-  }
-}
-
-void print_received_command(
-    uint16_t command,
-    uint32_t flag,
-    const void* data,
-    size_t size,
-    GameVersion version,
-    const char* name,
-    TerminalFormat color) {
-  if (use_terminal_colors) {
-    print_color_escape(stderr, color, TerminalFormat::BOLD, TerminalFormat::END);
-  }
-
-  string name_token;
-  if (name && name[0]) {
-    name_token = string(" from ") + name;
-  }
-  log(INFO, "Received%s (version=%s command=%04hX flag=%08X)",
-      name_token.c_str(), name_for_version(version), command, flag);
-
-  PSOCommandHeader header;
-  size_t header_size = header.header_size(version);
-  header.set_command(version, command);
-  header.set_flag(version, flag);
-  header.set_size(version, size + header_size);
-
-  // TODO: This is unnecessarily slow. It'd be nice to have a print_data_v() so
-  // we don't have to copy data around here.
-  StringWriter w;
-  w.write(&header, header_size);
-  w.write(data, size);
-
-  print_data(stderr, w.str());
-
-  if (use_terminal_colors) {
-    print_color_escape(stderr, TerminalFormat::NORMAL, TerminalFormat::END);
-  }
-}
-
 void check_size_v(size_t size, size_t min_size, size_t max_size) {
   if (size < min_size) {
     throw std::runtime_error(string_printf(

src/PSOProtocol.hh

@@ -10,7 +10,7 @@
 #include "PSOEncryption.hh"
 
 struct PSOCommandHeaderPC {
-  uint16_t size;
+  le_uint16_t size;
   uint8_t command;
   uint8_t flag;
 } __attribute__((packed));
@@ -18,13 +18,13 @@ struct PSOCommandHeaderPC {
 struct PSOCommandHeaderDCGC {
   uint8_t command;
   uint8_t flag;
-  uint16_t size;
+  le_uint16_t size;
 } __attribute__((packed));
 
 struct PSOCommandHeaderBB {
-  uint16_t size;
-  uint16_t command;
-  uint32_t flag;
+  le_uint16_t size;
+  le_uint16_t command;
+  le_uint32_t flag;
 } __attribute__((packed));
 
 union PSOCommandHeader {
@@ -48,25 +48,10 @@ union PSOCommandHeader {
 
 union PSOSubcommand {
   uint8_t byte[4];
-  uint16_t word[2];
-  uint32_t dword;
+  le_uint16_t word[2];
+  le_uint32_t dword;
 } __attribute__((packed));
 
-void for_each_received_command(
-    struct bufferevent* bev,
-    GameVersion version,
-    PSOEncryption* crypt,
-    std::function<void(uint16_t, uint16_t, std::string&)> fn);
-
-void print_received_command(
-    uint16_t command,
-    uint32_t flag,
-    const void* data,
-    size_t size,
-    GameVersion version,
-    const char* name = nullptr,
-    TerminalFormat color = TerminalFormat::FG_GREEN);
-
 // This function is used in a lot of places to check received command sizes and
 // cast them to the appropriate type
 template <typename T>

src/Player.cc

@@ -299,6 +299,7 @@ shared_ptr<SavedAccountDataBB> ClientGameData::account(bool should_load) {
   if (!this->account_data.get() && should_load) {
     if (this->bb_username.empty()) {
       this->account_data.reset(new SavedAccountDataBB());
+      this->account_data->signature = ACCOUNT_FILE_SIGNATURE;
     } else {
       this->load_account_data();
     }
@@ -310,6 +311,7 @@ shared_ptr<SavedPlayerDataBB> ClientGameData::player(bool should_load) {
   if (!this->player_data.get() && should_load) {
     if (this->bb_username.empty()) {
       this->player_data.reset(new SavedPlayerDataBB());
+      this->player_data->signature = PLAYER_FILE_SIGNATURE;
     } else {
       this->load_player_data();
     }
@@ -348,7 +350,7 @@ string ClientGameData::player_data_filename() const {
 }
 
 string ClientGameData::player_template_filename(uint8_t char_class) {
-  return string_printf("system/blueburst/player_class_%hhu.nsc", char_class);
+  return string_printf("system/players/default_player_%hhu.nsc", char_class);
 }
 
 void ClientGameData::create_player(
@@ -386,7 +388,7 @@ void ClientGameData::load_account_data() {
     log(INFO, "[BB/Account] No account data for %s; using default",
         this->bb_username.c_str());
     data.reset(new SavedAccountDataBB(
-        load_object_file<SavedAccountDataBB>("system/blueburst/default.nsa")));
+        load_object_file<SavedAccountDataBB>("system/players/default.nsa")));
     if (data->signature != ACCOUNT_FILE_SIGNATURE) {
       throw runtime_error("default account data header is incorrect");
     }
@@ -487,7 +489,6 @@ PlayerBB ClientGameData::export_player_bb() {
   ret.unknown6.clear();
   ret.quest_data2 = player->quest_data2;
   ret.key_config = account->key_config;
-  ret.unused = 0;
   return ret;
 }
 
@@ -539,7 +540,7 @@ PlayerBankItem::PlayerBankItem()
 
 PlayerBankItem::PlayerBankItem(const PlayerInventoryItem& src)
   : data(src.data),
-    amount(combine_item_to_max.count(this->data.primary_identifier()) ? this->data.data1[5] : 1),
+    amount(stack_size_for_item(this->data)),
     show_flags(1) { }
 
 
@@ -568,9 +569,8 @@ void SavedPlayerDataBB::add_item(const PlayerInventoryItem& item) {
   }
 
   // Handle combinable items
-  try {
-    uint32_t combine_max = combine_item_to_max.at(pid);
-
+  size_t combine_max = stack_size_for_item(item.data);
+  if (combine_max > 1) {
     // Get the item index if there's already a stack of the same item in the
     // player's inventory
     size_t y;
@@ -588,7 +588,7 @@ void SavedPlayerDataBB::add_item(const PlayerInventoryItem& item) {
       }
       return;
     }
-  } catch (const out_of_range&) { }
+  }
 
   // If we get here, then it's not meseta and not a combine item, so it needs to
   // go into an empty inventory slot
@@ -610,9 +610,8 @@ void PlayerBank::add_item(const PlayerBankItem& item) {
     return;
   }
 
-  try {
-    uint32_t combine_max = combine_item_to_max.at(pid);
-
+  size_t combine_max = stack_size_for_item(item.data);
+  if (combine_max > 1) {
     size_t y;
     for (y = 0; y < this->num_items; y++) {
       if (this->items[y].data.primary_identifier() == item.data.primary_identifier()) {
@@ -628,7 +627,7 @@ void PlayerBank::add_item(const PlayerBankItem& item) {
       this->items[y].amount = this->items[y].data.data1[5];
       return;
     }
-  } catch (const out_of_range&) { }
+  }
 
   if (this->num_items >= 200) {
     throw runtime_error("bank is full");
@@ -662,8 +661,8 @@ PlayerInventoryItem SavedPlayerDataBB::remove_item(
   // then create a new item and reduce the amount of the existing stack. Note
   // that passing amount == 0 means to remove the entire stack, so this only
   // applies if amount is nonzero.
-  if (amount && (amount < inventory_item.data.data1[5]) &&
-      combine_item_to_max.count(inventory_item.data.primary_identifier())) {
+  if (amount && (stack_size_for_item(inventory_item.data) > 1) &&
+      (amount < inventory_item.data.data1[5])) {
     ret = inventory_item;
     ret.data.data1[5] = amount;
     ret.data.id = 0xFFFFFFFF;
@@ -699,8 +698,8 @@ PlayerBankItem PlayerBank::remove_item(uint32_t item_id, uint32_t amount) {
   size_t index = this->find_item(item_id);
   auto& bank_item = this->items[index];
 
-  if (amount && (amount < bank_item.data.data1[5]) &&
-      combine_item_to_max.count(bank_item.data.primary_identifier())) {
+  if (amount && (stack_size_for_item(bank_item.data) > 1) &&
+      (amount < bank_item.data.data1[5])) {
     ret = bank_item;
     ret.data.data1[5] = amount;
     ret.amount = amount;

src/Player.hh

@@ -10,6 +10,7 @@
 #include "LevelTable.hh"
 #include "Version.hh"
 #include "Text.hh"
+#include "Episode3.hh"
 
 
 
@@ -303,6 +304,17 @@ struct PSOPlayerDataGC { // For command 61
   char auto_reply[0];
 } __attribute__((packed));
 
+struct PSOPlayerDataGCEp3 { // For command 61
+  PlayerInventory inventory;
+  PlayerDispDataPCGC disp;
+  parray<uint8_t, 0x134> unknown;
+  ptext<char, 0xAC> info_board;
+  parray<le_uint32_t, 0x1E> blocked_senders;
+  le_uint32_t auto_reply_enabled;
+  char auto_reply[0xAC];
+  Ep3Config ep3_config;
+} __attribute__((packed));
+
 struct PSOPlayerDataBB { // For command 61
   PlayerInventory inventory;
   PlayerDispDataBB disp;
@@ -314,33 +326,32 @@ struct PSOPlayerDataBB { // For command 61
 } __attribute__((packed));
 
 struct PlayerBB { // Used in 00E7 command
-  PlayerInventory inventory;                // 0000 // player
-  PlayerDispDataBB disp;                    // 034C // player
-  parray<uint8_t, 0x0010> unknown;          // 04DC //
-  le_uint32_t option_flags;                 // 04EC // account
-  parray<uint8_t, 0x0208> quest_data1;      // 04F0 // player
-  PlayerBank bank;                          // 06F8 // player
-  le_uint32_t serial_number;                // 19C0 // player
-  ptext<char16_t, 0x18> name;               // 19C4 // player
-  ptext<char16_t, 0x10> team_name;          // 19C4 // player
-  ptext<char16_t, 0x58> guild_card_desc;    // 1A14 // player
-  uint8_t reserved1;                        // 1AC4 // player
-  uint8_t reserved2;                        // 1AC5 // player
-  uint8_t section_id;                       // 1AC6 // player
-  uint8_t char_class;                       // 1AC7 // player
-  le_uint32_t unknown3;                     // 1AC8 //
-  parray<uint8_t, 0x04E0> symbol_chats;     // 1ACC // account
-  parray<uint8_t, 0x0A40> shortcuts;        // 1FAC // account
-  ptext<char16_t, 0x00AC> auto_reply;       // 29EC // player
-  ptext<char16_t, 0x00AC> info_board;       // 2B44 // player
-  parray<uint8_t, 0x001C> unknown5;         // 2C9C //
-  parray<uint8_t, 0x0140> challenge_data;   // 2CB8 // player
-  parray<uint8_t, 0x0028> tech_menu_config; // 2DF8 // player
-  parray<uint8_t, 0x002C> unknown6;         // 2E20 //
-  parray<uint8_t, 0x0058> quest_data2;      // 2E4C // player
-  KeyAndTeamConfigBB key_config;            // 2EA4 // account
-  le_uint32_t unused;
-} __attribute__((packed));                  // total size: 39A0
+  PlayerInventory inventory;                // player
+  PlayerDispDataBB disp;                    // player
+  parray<uint8_t, 0x0010> unknown;          // not saved
+  le_uint32_t option_flags;                 // account
+  parray<uint8_t, 0x0208> quest_data1;      // player
+  PlayerBank bank;                          // player
+  le_uint32_t serial_number;                // player
+  ptext<char16_t, 0x18> name;               // player
+  ptext<char16_t, 0x10> team_name;          // player
+  ptext<char16_t, 0x58> guild_card_desc;    // player
+  uint8_t reserved1;                        // player
+  uint8_t reserved2;                        // player
+  uint8_t section_id;                       // player
+  uint8_t char_class;                       // player
+  le_uint32_t unknown3;                     // not saved
+  parray<uint8_t, 0x04E0> symbol_chats;     // account
+  parray<uint8_t, 0x0A40> shortcuts;        // account
+  ptext<char16_t, 0x00AC> auto_reply;       // player
+  ptext<char16_t, 0x00AC> info_board;       // player
+  parray<uint8_t, 0x001C> unknown5;         // not saved
+  parray<uint8_t, 0x0140> challenge_data;   // player
+  parray<uint8_t, 0x0028> tech_menu_config; // player
+  parray<uint8_t, 0x002C> unknown6;         // not saved
+  parray<uint8_t, 0x0058> quest_data2;      // player
+  KeyAndTeamConfigBB key_config;            // account
+} __attribute__((packed));
 
 
 
@@ -376,6 +387,8 @@ struct SavedAccountDataBB { // .nsa file format
   ptext<char16_t, 0x0010>     team_name;
 } __attribute__((packed));
 
+
+
 class ClientGameData {
 private:
   std::shared_ptr<SavedAccountDataBB> account_data;
@@ -387,7 +400,7 @@ public:
   // The following fields are not saved, and are only used in certain situations
 
   // Null unless the client is Episode 3 and has sent its config already
-  std::shared_ptr<parray<uint8_t, 0x2408>> ep3_config;
+  std::shared_ptr<Ep3Config> ep3_config;
 
   // These are only used if the client is BB
   std::string bb_username;

src/ProxyServer.cc

@@ -29,17 +29,12 @@
 #include "ProxyCommands.hh"
 
 using namespace std;
+using namespace std::placeholders;
 
 
 
-static const uint32_t SESSION_TIMEOUT_USECS = 10000000; // 10 seconds
-
-
-
-static void flush_and_free_bufferevent(struct bufferevent* bev) {
-  bufferevent_flush(bev, EV_READ | EV_WRITE, BEV_FINISHED);
-  bufferevent_free(bev);
-}
+static const uint32_t LICENSED_SESSION_TIMEOUT_USECS = 5 * 60 * 1000000; // 5 minutes
+static const uint32_t UNLICENSED_SESSION_TIMEOUT_USECS = 10 * 1000000; // 10 seconds
 
 
 
@@ -133,7 +128,8 @@ void ProxyServer::connect_client(struct bufferevent* bev, uint16_t server_port)
   } catch (const out_of_range&) {
     this->log(INFO, "Virtual connection received on unregistered port %hu; closing it",
         server_port);
-    flush_and_free_bufferevent(bev);
+    bufferevent_flush(bev, EV_READ | EV_WRITE, BEV_FINISHED);
+    bufferevent_free(bev);
     return;
   }
 
@@ -165,7 +161,9 @@ void ProxyServer::on_client_connect(
     }
     auto session = emplace_ret.first->second;
     session->log(INFO, "Opened linked session");
-    session->resume(bev);
+
+    Channel ch(bev, version, nullptr, nullptr, session.get(), "", TerminalFormat::FG_YELLOW, TerminalFormat::FG_GREEN);
+    session->resume(move(ch));
 
   // If no default destination exists, or the client is not a patch client,
   // create an unlinked session - we'll have to get the destination from the
@@ -194,22 +192,15 @@ void ProxyServer::on_client_connect(
         uint32_t client_key = random_object<uint32_t>();
         auto cmd = prepare_server_init_contents_dc_pc_gc(
             false, server_key, client_key);
-        send_command(
-            session->bev.get(),
-            session->version,
-            session->crypt_out.get(),
-            0x02,
-            0,
-            &cmd,
-            sizeof(cmd),
-            "unlinked proxy client");
-        bufferevent_flush(session->bev.get(), EV_READ | EV_WRITE, BEV_FLUSH);
+        session->channel.send(0x02, 0x00, &cmd, sizeof(cmd));
+        // TODO: Is this actually needed?
+        // bufferevent_flush(session->channel.bev.get(), EV_READ | EV_WRITE, BEV_FLUSH);
         if (version == GameVersion::PC) {
-          session->crypt_out.reset(new PSOPCEncryption(server_key));
-          session->crypt_in.reset(new PSOPCEncryption(client_key));
+          session->channel.crypt_out.reset(new PSOPCEncryption(server_key));
+          session->channel.crypt_in.reset(new PSOPCEncryption(client_key));
         } else {
-          session->crypt_out.reset(new PSOGCEncryption(server_key));
-          session->crypt_in.reset(new PSOGCEncryption(client_key));
+          session->channel.crypt_out.reset(new PSOGCEncryption(server_key));
+          session->channel.crypt_in.reset(new PSOGCEncryption(client_key));
         }
         break;
       }
@@ -218,23 +209,15 @@ void ProxyServer::on_client_connect(
         parray<uint8_t, 0x30> client_key;
         random_data(server_key.data(), server_key.bytes());
         random_data(client_key.data(), client_key.bytes());
-        auto cmd = prepare_server_init_contents_bb(server_key, client_key);
-        send_command(
-            session->bev.get(),
-            session->version,
-            session->crypt_out.get(),
-            0x03,
-            0,
-            &cmd,
-            sizeof(cmd),
-            "unlinked proxy client");
-        bufferevent_flush(session->bev.get(), EV_READ | EV_WRITE, BEV_FLUSH);
-
+        auto cmd = prepare_server_init_contents_bb(server_key, client_key, false);
+        session->channel.send(0x03, 0x00, &cmd, sizeof(cmd));
+        // TODO: Is this actually needed?
+        // bufferevent_flush(session->bev.get(), EV_READ | EV_WRITE, BEV_FLUSH);
         static const string expected_first_data("\xB4\x00\x93\x00\x00\x00\x00\x00", 8);
         session->detector_crypt.reset(new PSOBBMultiKeyDetectorEncryption(
             this->state->bb_private_keys, expected_first_data, cmd.client_key.data(), sizeof(cmd.client_key)));
-        session->crypt_in = session->detector_crypt;
-        session->crypt_out.reset(new PSOBBMultiKeyImitatorEncryption(
+        session->channel.crypt_in = session->detector_crypt;
+        session->channel.crypt_out.reset(new PSOBBMultiKeyImitatorEncryption(
             session->detector_crypt, cmd.server_key.data(), sizeof(cmd.server_key), true));
         break;
       }
@@ -250,87 +233,79 @@ ProxyServer::UnlinkedSession::UnlinkedSession(
     ProxyServer* server, struct bufferevent* bev, uint16_t local_port, GameVersion version)
   : server(server),
     log(string_printf("[ProxyServer:UnlinkedSession:%p] ", bev)),
-    bev(bev, flush_and_free_bufferevent),
+    channel(
+      bev,
+      version,
+      ProxyServer::UnlinkedSession::on_input,
+      ProxyServer::UnlinkedSession::on_error,
+      this,
+      string_printf("UnlinkedSession:%p", bev),
+      TerminalFormat::FG_YELLOW,
+      TerminalFormat::FG_GREEN),
     local_port(local_port),
     version(version) {
   memset(&this->next_destination, 0, sizeof(this->next_destination));
-  bufferevent_setcb(this->bev.get(),
-      &UnlinkedSession::dispatch_on_client_input, nullptr,
-      &UnlinkedSession::dispatch_on_client_error, this);
-  bufferevent_enable(this->bev.get(), EV_READ | EV_WRITE);
 }
 
-void ProxyServer::UnlinkedSession::dispatch_on_client_input(
-    struct bufferevent*, void* ctx) {
-  reinterpret_cast<UnlinkedSession*>(ctx)->on_client_input();
-}
+void ProxyServer::UnlinkedSession::on_input(Channel& ch, uint16_t command, uint32_t, std::string& data) {
+  auto* session = reinterpret_cast<UnlinkedSession*>(ch.context_obj);
 
-void ProxyServer::UnlinkedSession::dispatch_on_client_error(
-    struct bufferevent*, short events, void* ctx) {
-  reinterpret_cast<UnlinkedSession*>(ctx)->on_client_error(events);
-}
-
-void ProxyServer::UnlinkedSession::on_client_input() {
   bool should_close_unlinked_session = false;
   shared_ptr<const License> license;
   uint32_t sub_version = 0;
   string character_name;
   ClientConfigBB client_config;
-  C_Login_BB_93 login_command_bb;
+  string login_command_bb;
 
   try {
-    for_each_received_command(this->bev.get(), this->version, this->crypt_in.get(),
-      [&](uint16_t command, uint32_t flag, const string& data) {
-        print_received_command(command, flag, data.data(), data.size(),
-            this->version, "unlinked proxy client");
+    if (session->version == GameVersion::PC) {
+      // We should only get a 9D while the session is unlinked; if we get
+      // anything else, disconnect
+      if (command != 0x9D) {
+        throw runtime_error("command is not 9D");
+      }
+      const auto& cmd = check_size_t<C_Login_PC_9D>(
+          data, sizeof(C_Login_PC_9D), sizeof(C_LoginExtended_PC_9D));
+      license = session->server->state->license_manager->verify_pc(
+          stoul(cmd.serial_number, nullptr, 16), cmd.access_key);
+      sub_version = cmd.sub_version;
+      character_name = cmd.name;
 
-        if (this->version == GameVersion::PC) {
-          // We should only get a 9D while the session is unlinked; if we get
-          // anything else, disconnect
-          if (command != 0x9D) {
-            throw runtime_error("command is not 9D");
-          }
-          const auto& cmd = check_size_t<C_Login_PC_9D>(data, sizeof(C_Login_PC_9D), sizeof(C_LoginWithUnusedSpace_PC_9D));
-          license = this->server->state->license_manager->verify_pc(
-              stoul(cmd.serial_number, nullptr, 16), cmd.access_key);
-          sub_version = cmd.sub_version;
-          character_name = cmd.name;
+    } else if (session->version == GameVersion::GC) {
+      // We should only get a 9E while the session is unlinked; if we get
+      // anything else, disconnect
+      if (command != 0x9E) {
+        throw runtime_error("command is not 9E");
+      }
+      const auto& cmd = check_size_t<C_Login_GC_9E>(
+          data, sizeof(C_Login_GC_9E), sizeof(C_LoginExtended_GC_9E));
+      license = session->server->state->license_manager->verify_gc(
+          stoul(cmd.serial_number, nullptr, 16), cmd.access_key);
+      sub_version = cmd.sub_version;
+      character_name = cmd.name;
+      client_config.cfg = cmd.client_config.cfg;
 
-        } else if (this->version == GameVersion::GC) {
-          // We should only get a 9E while the session is unlinked; if we get
-          // anything else, disconnect
-          if (command != 0x9E) {
-            throw runtime_error("command is not 9E");
-          }
-          const auto& cmd = check_size_t<C_Login_GC_9E>(data, sizeof(C_Login_GC_9E), sizeof(C_LoginWithUnusedSpace_GC_9E));
-          license = this->server->state->license_manager->verify_gc(
-              stoul(cmd.serial_number, nullptr, 16), cmd.access_key);
-          sub_version = cmd.sub_version;
-          character_name = cmd.name;
-          client_config.cfg = cmd.client_config.cfg;
+    } else if (session->version == GameVersion::BB) {
+      // We should only get a 93 while the session is unlinked; if we get
+      // anything else, disconnect
+      if (command != 0x93) {
+        throw runtime_error("command is not 93");
+      }
+      const auto& cmd = check_size_t<C_Login_BB_93>(data);
+      license = session->server->state->license_manager->verify_bb(
+          cmd.username, cmd.password);
+      login_command_bb = move(data);
 
-        } else if (this->version == GameVersion::BB) {
-          // We should only get a 93 while the session is unlinked; if we get
-          // anything else, disconnect
-          if (command != 0x93) {
-            throw runtime_error("command is not 93");
-          }
-          const auto& cmd = check_size_t<C_Login_BB_93>(data);
-          license = this->server->state->license_manager->verify_bb(
-              cmd.username, cmd.password);
-          login_command_bb = cmd;
-
-        } else {
-          throw logic_error("unsupported unlinked session version");
-        }
-      });
+    } else {
+      throw logic_error("unsupported unlinked session version");
+    }
 
   } catch (const exception& e) {
-    this->log(ERROR, "Failed to process command from unlinked client: %s", e.what());
+    session->log(ERROR, "Failed to process command from unlinked client: %s", e.what());
     should_close_unlinked_session = true;
   }
 
-  struct bufferevent* session_key = this->bev.get();
+  struct bufferevent* session_key = ch.bev.get();
 
   // If license is non-null, then the client has a password and can be connected
   // to the remote lobby server.
@@ -340,75 +315,80 @@ void ProxyServer::UnlinkedSession::on_client_input() {
     should_close_unlinked_session = true;
 
     // Look up the linked session for this license (if any)
-    shared_ptr<LinkedSession> session;
+    shared_ptr<LinkedSession> linked_session;
     try {
-      session = this->server->id_to_session.at(license->serial_number);
-      session->log(INFO, "Resuming linked session from unlinked session");
+      linked_session = session->server->id_to_session.at(license->serial_number);
+      linked_session->log(INFO, "Resuming linked session from unlinked session");
 
     } catch (const out_of_range&) {
       // If there's no open session for this license, then there must be a valid
       // destination somewhere - either in the client config or in the unlinked
       // session
       if (client_config.cfg.magic == CLIENT_CONFIG_MAGIC) {
-        session.reset(new LinkedSession(
-            this->server,
-            this->local_port,
-            this->version,
+        linked_session.reset(new LinkedSession(
+            session->server,
+            session->local_port,
+            session->version,
             license,
             client_config));
-        session->log(INFO, "Opened licensed session for unlinked session based on client config");
-      } else if (this->next_destination.ss_family == AF_INET) {
-        session.reset(new LinkedSession(
-            this->server,
-            this->local_port,
-            this->version,
+        linked_session->log(INFO, "Opened licensed session for unlinked session based on client config");
+      } else if (session->next_destination.ss_family == AF_INET) {
+        linked_session.reset(new LinkedSession(
+            session->server,
+            session->local_port,
+            session->version,
             license,
-            this->next_destination));
-        session->log(INFO, "Opened licensed session for unlinked session based on unlinked default destination");
+            session->next_destination));
+        linked_session->log(INFO, "Opened licensed session for unlinked session based on unlinked default destination");
       } else {
-        this->log(ERROR, "Cannot open linked session: no valid destination in client config or unlinked session");
+        session->log(ERROR, "Cannot open linked session: no valid destination in client config or unlinked session");
       }
     }
 
-    if (session.get()) {
-      this->server->id_to_session.emplace(license->serial_number, session);
-      if (session->version != this->version) {
-        session->log(ERROR, "Linked session has different game version");
+    if (linked_session.get()) {
+      session->server->id_to_session.emplace(license->serial_number, linked_session);
+      if (linked_session->version != session->version) {
+        linked_session->log(ERROR, "Linked session has different game version");
       } else {
         // Resume the linked session using the unlinked session
         try {
-          if (this->version == GameVersion::BB) {
-            session->resume(move(this->bev), this->crypt_in, this->crypt_out,
-                this->detector_crypt, login_command_bb);
+          if (session->version == GameVersion::BB) {
+            linked_session->resume(
+                move(session->channel),
+                session->detector_crypt,
+                move(login_command_bb));
           } else {
-            session->resume(move(this->bev), this->crypt_in, this->crypt_out,
-                this->detector_crypt, sub_version, character_name);
+            linked_session->resume(
+                move(session->channel),
+                session->detector_crypt,
+                sub_version,
+                character_name);
           }
-          this->crypt_in.reset();
-          this->crypt_out.reset();
         } catch (const exception& e) {
-          session->log(ERROR, "Failed to resume linked session: %s", e.what());
+          linked_session->log(ERROR, "Failed to resume linked session: %s", e.what());
         }
       }
     }
   }
 
   if (should_close_unlinked_session) {
-    this->log(INFO, "Closing session");
-    this->server->bev_to_unlinked_session.erase(session_key);
+    session->log(INFO, "Closing session");
+    session->server->bev_to_unlinked_session.erase(session_key);
     // At this point, (*this) is destroyed! We must be careful not to touch it.
   }
 }
 
-void ProxyServer::UnlinkedSession::on_client_error(short events) {
+void ProxyServer::UnlinkedSession::on_error(Channel& ch, short events) {
+  auto* session = reinterpret_cast<UnlinkedSession*>(ch.context_obj);
+
   if (events & BEV_EVENT_ERROR) {
     int err = EVUTIL_SOCKET_ERROR();
-    this->log(WARNING, "Error %d (%s) in unlinked client stream", err,
+    session->log(WARNING, "Error %d (%s) in unlinked client stream", err,
         evutil_socket_error_to_string(err));
   }
   if (events & (BEV_EVENT_ERROR | BEV_EVENT_EOF)) {
-    this->log(WARNING, "Unlinked client has disconnected");
-    this->server->bev_to_unlinked_session.erase(this->bev.get());
+    session->log(WARNING, "Unlinked client has disconnected");
+    session->server->bev_to_unlinked_session.erase(session->channel.bev.get());
   }
 }
 
@@ -421,21 +401,36 @@ ProxyServer::LinkedSession::LinkedSession(
     GameVersion version)
   : server(server),
     id(id),
-    client_name(string_printf("LinkedSession:%08" PRIX64 ":client", this->id)),
-    server_name(string_printf("LinkedSession:%08" PRIX64 ":server", this->id)),
     log(string_printf("[ProxyServer:LinkedSession:%08" PRIX64 "] ", this->id)),
     timeout_event(event_new(this->server->base.get(), -1, EV_TIMEOUT,
         &LinkedSession::dispatch_on_timeout, this), event_free),
     license(nullptr),
-    client_bev(nullptr, flush_and_free_bufferevent),
-    server_bev(nullptr, flush_and_free_bufferevent),
+    client_channel(
+      version,
+      nullptr,
+      nullptr,
+      this,
+      string_printf("LinkedSession:%08" PRIX64 ":client", this->id),
+      TerminalFormat::FG_YELLOW,
+      TerminalFormat::FG_GREEN),
+    server_channel(
+      version,
+      nullptr,
+      nullptr,
+      this,
+      string_printf("LinkedSession:%08" PRIX64 ":server", this->id),
+      TerminalFormat::FG_YELLOW,
+      TerminalFormat::FG_RED),
     local_port(local_port),
+    remote_ip_crc(0),
+    enable_remote_ip_crc_patch(false),
     version(version),
     sub_version(0), // This is set during resume()
     remote_guild_card_number(0),
-    suppress_newserv_commands(true),
     enable_chat_filter(true),
-    enable_switch_assist(false),
+    switch_assist(false),
+    infinite_hp(false),
+    infinite_tp(false),
     save_files(false),
     function_call_return_value(-1),
     override_section_id(-1),
@@ -485,60 +480,48 @@ ProxyServer::LinkedSession::LinkedSession(
 }
 
 void ProxyServer::LinkedSession::resume(
-    unique_ptr<struct bufferevent, void(*)(struct bufferevent*)>&& client_bev,
-    shared_ptr<PSOEncryption> client_input_crypt,
-    shared_ptr<PSOEncryption> client_output_crypt,
+    Channel&& client_channel,
     shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt,
     uint32_t sub_version,
     const string& character_name) {
   this->sub_version = sub_version;
   this->character_name = character_name;
-  this->resume_inner(move(client_bev), client_input_crypt, client_output_crypt,
-      detector_crypt);
+  this->resume_inner(move(client_channel), detector_crypt);
 }
 
 void ProxyServer::LinkedSession::resume(
-    unique_ptr<struct bufferevent, void(*)(struct bufferevent*)>&& client_bev,
-    shared_ptr<PSOEncryption> client_input_crypt,
-    shared_ptr<PSOEncryption> client_output_crypt,
+    Channel&& client_channel,
     shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt,
-    C_Login_BB_93 login_command_bb) {
-  this->login_command_bb = login_command_bb;
-  this->resume_inner(move(client_bev), client_input_crypt, client_output_crypt,
-      detector_crypt);
+    string&& login_command_bb) {
+  this->login_command_bb = move(login_command_bb);
+  this->resume_inner(move(client_channel), detector_crypt);
 }
 
-void ProxyServer::LinkedSession::resume(struct bufferevent* client_bev) {
-  unique_ptr<struct bufferevent, void(*)(struct bufferevent*)> bev_unique(
-      client_bev, flush_and_free_bufferevent);
+void ProxyServer::LinkedSession::resume(Channel&& client_channel) {
   this->sub_version = 0;
   this->character_name.clear();
-  this->resume_inner(move(bev_unique), nullptr, nullptr, nullptr);
+  this->resume_inner(move(client_channel), nullptr);
 }
 
 void ProxyServer::LinkedSession::resume_inner(
-    unique_ptr<struct bufferevent, void(*)(struct bufferevent*)>&& client_bev,
-    shared_ptr<PSOEncryption> client_input_crypt,
-    shared_ptr<PSOEncryption> client_output_crypt,
+    Channel&& client_channel,
     shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt) {
-  if (this->client_bev.get()) {
+  if (this->client_channel.connected()) {
     throw runtime_error("client connection is already open for this session");
   }
   if (this->next_destination.ss_family != AF_INET) {
     throw logic_error("attempted to resume an unlicensed linked session without destination set");
   }
 
-  this->client_bev = move(client_bev);
-  bufferevent_setcb(this->client_bev.get(),
-      &ProxyServer::LinkedSession::dispatch_on_client_input, nullptr,
-      &ProxyServer::LinkedSession::dispatch_on_client_error, this);
-  bufferevent_enable(this->client_bev.get(), EV_READ | EV_WRITE);
+  this->client_channel.replace_with(
+      move(client_channel),
+      ProxyServer::LinkedSession::on_input,
+      ProxyServer::LinkedSession::on_error,
+      this,
+      string_printf("LinkedSession:%08" PRIX64 ":client", this->id));
 
   this->detector_crypt = detector_crypt;
-  this->client_input_crypt = client_input_crypt;
-  this->client_output_crypt = client_output_crypt;
-  this->server_input_crypt.reset();
-  this->server_output_crypt.reset();
+  this->server_channel.disconnect();
   this->saving_files.clear();
 
   this->connect();
@@ -547,9 +530,6 @@ void ProxyServer::LinkedSession::resume_inner(
 void ProxyServer::LinkedSession::connect() {
   // Connect to the remote server. The command handlers will do the login steps
   // and set up forwarding
-  this->server_bev.reset(bufferevent_socket_new(this->server->base.get(), -1,
-      BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS));
-
   struct sockaddr_storage local_ss;
   struct sockaddr_in* local_sin = reinterpret_cast<struct sockaddr_in*>(&local_ss);
   memset(local_sin, 0, sizeof(*local_sin));
@@ -563,14 +543,17 @@ void ProxyServer::LinkedSession::connect() {
 
   string netloc_str = render_sockaddr_storage(local_ss);
   this->log(INFO, "Connecting to %s", netloc_str.c_str());
-  if (bufferevent_socket_connect(this->server_bev.get(),
+
+  this->server_channel.set_bufferevent(bufferevent_socket_new(
+      this->server->base.get(), -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS));
+  if (bufferevent_socket_connect(this->server_channel.bev.get(),
       reinterpret_cast<const sockaddr*>(local_sin), sizeof(*local_sin)) != 0) {
     throw runtime_error(string_printf("failed to connect (%d)", EVUTIL_SOCKET_ERROR()));
   }
-  bufferevent_setcb(this->server_bev.get(),
-      &ProxyServer::LinkedSession::dispatch_on_server_input, nullptr,
-      &ProxyServer::LinkedSession::dispatch_on_server_error, this);
-  bufferevent_enable(this->server_bev.get(), EV_READ | EV_WRITE);
+
+  this->server_channel.on_command_received = ProxyServer::LinkedSession::on_input;
+  this->server_channel.on_error = ProxyServer::LinkedSession::on_error;
+  this->server_channel.context_obj = this;
 
   // Cancel the session delete timeout
   event_del(this->timeout_event.get());
@@ -589,26 +572,6 @@ ProxyServer::LinkedSession::SavingFile::SavingFile(
 
 
 
-void ProxyServer::LinkedSession::dispatch_on_client_input(
-    struct bufferevent*, void* ctx) {
-  reinterpret_cast<LinkedSession*>(ctx)->on_client_input();
-}
-
-void ProxyServer::LinkedSession::dispatch_on_client_error(
-    struct bufferevent*, short events, void* ctx) {
-  reinterpret_cast<LinkedSession*>(ctx)->on_stream_error(events, false);
-}
-
-void ProxyServer::LinkedSession::dispatch_on_server_input(
-    struct bufferevent*, void* ctx) {
-  reinterpret_cast<LinkedSession*>(ctx)->on_server_input();
-}
-
-void ProxyServer::LinkedSession::dispatch_on_server_error(
-    struct bufferevent*, short events, void* ctx) {
-  reinterpret_cast<LinkedSession*>(ctx)->on_stream_error(events, true);
-}
-
 void ProxyServer::LinkedSession::dispatch_on_timeout(
     evutil_socket_t, short, void* ctx) {
   reinterpret_cast<LinkedSession*>(ctx)->on_timeout();
@@ -623,134 +586,64 @@ void ProxyServer::LinkedSession::on_timeout() {
 
 
 
-void ProxyServer::LinkedSession::on_stream_error(
-    short events, bool is_server_stream) {
+void ProxyServer::LinkedSession::on_error(Channel& ch, short events) {
+  auto* session = reinterpret_cast<LinkedSession*>(ch.context_obj);
+  bool is_server_stream = (&ch == &session->server_channel);
+
   if (events & BEV_EVENT_ERROR) {
     int err = EVUTIL_SOCKET_ERROR();
-    this->log(WARNING, "Error %d (%s) in %s stream",
+    session->log(WARNING, "Error %d (%s) in %s stream",
         err, evutil_socket_error_to_string(err),
         is_server_stream ? "server" : "client");
   }
   if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
-    this->log(INFO, "%s has disconnected",
+    session->log(INFO, "%s has disconnected",
         is_server_stream ? "Server" : "Client");
-    this->disconnect();
+    session->disconnect();
   }
 }
 
 void ProxyServer::LinkedSession::disconnect() {
   // Forward the disconnection to the other end
-  this->server_bev.reset();
-  this->client_bev.reset();
-
-  // Disable encryption for the next connection
-  this->server_input_crypt.reset();
-  this->server_output_crypt.reset();
-  this->client_input_crypt.reset();
-  this->client_output_crypt.reset();
+  this->client_channel.disconnect();
+  this->server_channel.disconnect();
 
   // Set a timeout to delete the session entirely (in case the client doesn't
   // reconnect)
-  struct timeval tv = usecs_to_timeval(SESSION_TIMEOUT_USECS);
+  struct timeval tv = usecs_to_timeval(this->license.get()
+      ? LICENSED_SESSION_TIMEOUT_USECS : UNLICENSED_SESSION_TIMEOUT_USECS);
   event_add(this->timeout_event.get(), &tv);
 }
 
+bool ProxyServer::LinkedSession::is_connected() const {
+  return (this->server_channel.connected() && this->client_channel.connected());
+}
 
 
-void ProxyServer::LinkedSession::on_client_input() {
+
+void ProxyServer::LinkedSession::on_input(Channel& ch, uint16_t command, uint32_t flag, std::string& data) {
+  auto* session = reinterpret_cast<LinkedSession*>(ch.context_obj);
+  bool is_server_stream = (&ch == &session->server_channel);
+
   try {
-    for_each_received_command(this->client_bev.get(), this->version, this->client_input_crypt.get(),
-      [&](uint16_t command, uint32_t flag, string& data) {
-        print_received_command(command, flag, data.data(), data.size(),
-            this->version, this->client_name.c_str());
-        process_proxy_command(
-            this->server->state,
-            *this,
-            false, // from_server
-            command,
-            flag,
-            data);
-      });
+    if (is_server_stream) {
+      size_t bytes_to_save = min<size_t>(data.size(), sizeof(session->prev_server_command_bytes));
+      memcpy(session->prev_server_command_bytes, data.data(), bytes_to_save);
+    }
+    process_proxy_command(
+        session->server->state,
+        *session,
+        is_server_stream,
+        command,
+        flag,
+        data);
   } catch (const exception& e) {
-    this->log(ERROR, "Failed to process command from client: %s", e.what());
-    this->disconnect();
+    session->log(ERROR, "Failed to process command from %s: %s",
+        is_server_stream ? "server" : "client", e.what());
+    session->disconnect();
   }
 }
 
-void ProxyServer::LinkedSession::on_server_input() {
-  try {
-    for_each_received_command(this->server_bev.get(), this->version, this->server_input_crypt.get(),
-      [&](uint16_t command, uint32_t flag, string& data) {
-        print_received_command(command, flag, data.data(), data.size(),
-            this->version, this->server_name.c_str(), TerminalFormat::FG_RED);
-        size_t bytes_to_save = min<size_t>(data.size(), sizeof(this->prev_server_command_bytes));
-        memcpy(this->prev_server_command_bytes, data.data(), bytes_to_save);
-        process_proxy_command(
-            this->server->state,
-            *this,
-            true, // from_server
-            command,
-            flag,
-            data);
-      });
-  } catch (const exception& e) {
-    this->log(ERROR, "Failed to process command from server: %s", e.what());
-    this->disconnect();
-  }
-}
-
-void ProxyServer::LinkedSession::send_to_end(
-    bool to_server,
-    uint16_t command,
-    uint32_t flag,
-    const void* data,
-    size_t size) {
-  if (size & 3) {
-    throw runtime_error("command size is not a multiple of 4");
-  }
-  string name = string_printf("LinkedSession:%08" PRIX64 ":synthetic:%s",
-      this->id, to_server ? "server" : "client");
-
-  auto* bev = to_server ? this->server_bev.get() : this->client_bev.get();
-  if (!bev) {
-    throw runtime_error("session endpoint is not connected");
-  }
-  send_command(
-      bev,
-      this->version,
-      to_server ? this->server_output_crypt.get() : this->client_output_crypt.get(),
-      command,
-      flag,
-      data,
-      size,
-      name.c_str());
-}
-
-void ProxyServer::LinkedSession::send_to_end(
-    bool to_server, uint16_t command, uint32_t flag, const string& data) {
-  this->send_to_end(to_server, command, flag, data.data(), data.size());
-}
-
-void ProxyServer::LinkedSession::send_to_end_with_header(
-    bool to_server, const void* data, size_t size) {
-  size_t header_size = PSOCommandHeader::header_size(this->version);
-  if (size < header_size) {
-    throw runtime_error("command is too small for header");
-  }
-  const auto* header = reinterpret_cast<const PSOCommandHeader*>(data);
-  this->send_to_end(
-      to_server,
-      header->command(this->version),
-      header->flag(this->version),
-      reinterpret_cast<const uint8_t*>(data) + header_size,
-      size - header_size);
-}
-
-void ProxyServer::LinkedSession::send_to_end_with_header(
-    bool to_server, const string& data) {
-  this->send_to_end_with_header(to_server, data.data(), data.size());
-}
-
 shared_ptr<ProxyServer::LinkedSession> ProxyServer::get_session() {
   if (this->id_to_session.empty()) {
     throw runtime_error("no sessions exist");
@@ -774,9 +667,21 @@ shared_ptr<ProxyServer::LinkedSession> ProxyServer::create_licensed_session(
   return emplace_ret.first->second;
 }
 
-
 void ProxyServer::delete_session(uint64_t id) {
   if (this->id_to_session.erase(id)) {
     this->log(INFO, "Closed LinkedSession:%08" PRIX64, id);
   }
 }
+
+size_t ProxyServer::delete_disconnected_sessions() {
+  size_t count = 0;
+  for (auto it = this->id_to_session.begin(); it != this->id_to_session.end();) {
+    if (!it->second->is_connected()) {
+      it = this->id_to_session.erase(it);
+      count++;
+    } else {
+      it++;
+    }
+  }
+  return count;
+}

src/ProxyServer.hh

@@ -35,32 +35,33 @@ public:
   struct LinkedSession {
     ProxyServer* server;
     uint64_t id;
-    std::string client_name;
-    std::string server_name;
     PrefixedLogger log;
 
     std::unique_ptr<struct event, void(*)(struct event*)> timeout_event;
 
     std::shared_ptr<const License> license;
 
-    std::unique_ptr<struct bufferevent, void(*)(struct bufferevent*)> client_bev;
-    std::unique_ptr<struct bufferevent, void(*)(struct bufferevent*)> server_bev;
+    Channel client_channel;
+    Channel server_channel;
     uint16_t local_port;
     struct sockaddr_storage next_destination;
 
     uint8_t prev_server_command_bytes[6];
+    uint32_t remote_ip_crc;
+    bool enable_remote_ip_crc_patch;
 
     GameVersion version;
     uint32_t sub_version;
     std::string character_name;
-    C_Login_BB_93 login_command_bb;
+    std::string login_command_bb;
 
     uint32_t remote_guild_card_number;
     parray<uint8_t, 0x20> remote_client_config_data;
     ClientConfigBB newserv_client_config;
-    bool suppress_newserv_commands;
     bool enable_chat_filter;
-    bool enable_switch_assist;
+    bool switch_assist;
+    bool infinite_hp;
+    bool infinite_tp;
     bool save_files;
     int64_t function_call_return_value; // -1 = don't block function calls
     G_SwitchStateChanged_6x05 last_switch_enabled_command;
@@ -76,10 +77,6 @@ public:
     std::vector<LobbyPlayer> lobby_players;
     size_t lobby_client_id;
 
-    std::shared_ptr<PSOEncryption> client_input_crypt;
-    std::shared_ptr<PSOEncryption> client_output_crypt;
-    std::shared_ptr<PSOEncryption> server_input_crypt;
-    std::shared_ptr<PSOEncryption> server_output_crypt;
     std::shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt;
 
     struct SavingFile {
@@ -121,49 +118,28 @@ public:
         const struct sockaddr_storage& next_destination);
 
     void resume(
-        std::unique_ptr<struct bufferevent, void(*)(struct bufferevent*)>&& client_bev,
-        std::shared_ptr<PSOEncryption> client_input_crypt,
-        std::shared_ptr<PSOEncryption> client_output_crypt,
+        Channel&& client_channel,
         std::shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt,
         uint32_t sub_version,
         const std::string& character_name);
     void resume(
-        std::unique_ptr<struct bufferevent, void(*)(struct bufferevent*)>&& client_bev,
-        std::shared_ptr<PSOEncryption> client_input_crypt,
-        std::shared_ptr<PSOEncryption> client_output_crypt,
+        Channel&& client_channel,
         std::shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt,
-        C_Login_BB_93 login_command_bb);
-    void resume(struct bufferevent* client_bev);
+        std::string&& login_command_bb);
+    void resume(Channel&& client_channel);
     void resume_inner(
-        std::unique_ptr<struct bufferevent, void(*)(struct bufferevent*)>&& client_bev,
-        std::shared_ptr<PSOEncryption> client_input_crypt,
-        std::shared_ptr<PSOEncryption> client_output_crypt,
+        Channel&& client_channel,
         std::shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt);
     void connect();
 
-    static void dispatch_on_client_input(struct bufferevent* bev, void* ctx);
-    static void dispatch_on_client_error(struct bufferevent* bev, short events,
-        void* ctx);
-    static void dispatch_on_server_input(struct bufferevent* bev, void* ctx);
-    static void dispatch_on_server_error(struct bufferevent* bev, short events,
-        void* ctx);
     static void dispatch_on_timeout(evutil_socket_t fd, short what, void* ctx);
-    void on_client_input();
-    void on_server_input();
-    void on_stream_error(short events, bool is_server_stream);
+    static void on_input(Channel& ch, uint16_t, uint32_t, std::string& msg);
+    static void on_error(Channel& ch, short events);
     void on_timeout();
 
-    void send_to_end(bool to_server, uint16_t command, uint32_t flag,
-        const void* data = nullptr, size_t size = 0);
-    void send_to_end(bool to_server, uint16_t command, uint32_t flag,
-        const std::string& data);
-    void send_to_end_with_header(
-        bool to_server, const void* data, size_t size);
-    void send_to_end_with_header(bool to_server, const std::string& data);
-
     void disconnect();
 
-    bool is_open() const;
+    bool is_connected() const;
   };
 
   std::shared_ptr<LinkedSession> get_session();
@@ -174,6 +150,8 @@ public:
     const ClientConfigBB& newserv_client_config);
   void delete_session(uint64_t id);
 
+  size_t delete_disconnected_sessions();
+
 private:
   struct ListeningSocket {
     ProxyServer* server;
@@ -202,24 +180,19 @@ private:
     ProxyServer* server;
 
     PrefixedLogger log;
-    std::unique_ptr<struct bufferevent, void(*)(struct bufferevent*)> bev;
+    Channel channel;
     uint16_t local_port;
     GameVersion version;
     struct sockaddr_storage next_destination;
 
-    std::shared_ptr<PSOEncryption> crypt_out;
-    std::shared_ptr<PSOEncryption> crypt_in;
     std::shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt;
 
     UnlinkedSession(ProxyServer* server, struct bufferevent* bev, uint16_t port, GameVersion version);
 
     void receive_and_process_commands();
 
-    static void dispatch_on_client_input(struct bufferevent* bev, void* ctx);
-    static void dispatch_on_client_error(struct bufferevent* bev, short events,
-        void* ctx);
-    void on_client_input();
-    void on_client_error(short events);
+    static void on_input(Channel& ch, uint16_t command, uint32_t flag, std::string& msg);
+    static void on_error(Channel& ch, short events);
   };
 
   PrefixedLogger log;

src/Quest.cc

@@ -68,6 +68,21 @@ const char* name_for_category(QuestCategory category) {
   }
 }
 
+static const char* name_for_episode(uint8_t episode) {
+  switch (episode) {
+    case 0:
+      return "Ep1";
+    case 1:
+      return "Ep2";
+    case 2:
+      return "Ep4";
+    case 0xFF:
+      return "Ep3";
+    default:
+      return "InvalidEpisode";
+  }
+}
+
 
 
 struct PSOQuestHeaderDC { // same for dc v1 and v2, thankfully
@@ -609,15 +624,9 @@ QuestIndex::QuestIndex(const std::string& directory) : directory(directory) {
             make_pair(q->version, q->menu_item_id), q).second) {
           throw logic_error("duplicate quest menu item id");
         }
-        if (!this->version_name_to_quest.emplace(
-            make_pair(q->version, q->name), q).second) {
-          throw runtime_error(string_printf(
-              "duplicate quest name (%s-%" PRId64 "): %s",
-              name_for_version(q->version), q->internal_id, ascii_name.c_str()));
-        }
-        log(INFO, "Indexed quest %s (%s-%" PRId64 " => %" PRIu32 ", %s, episode=%hhu, joinable=%s, dcv1=%s)",
+        log(INFO, "Indexed quest %s (%s-%" PRId64 " => %" PRIu32 ", %s, %s, joinable=%s, dcv1=%s)",
             ascii_name.c_str(), name_for_version(q->version), q->internal_id,
-            q->menu_item_id, name_for_category(q->category), q->episode,
+            q->menu_item_id, name_for_category(q->category), name_for_episode(q->episode),
             q->joinable ? "true" : "false", q->is_dcv1 ? "true" : "false");
       } catch (const exception& e) {
         log(WARNING, "Failed to parse quest file %s (%s)", filename.c_str(), e.what());

src/Quest.hh

@@ -35,11 +35,6 @@ const char* name_for_category(QuestCategory category);
 
 
 class Quest {
-private:
-  static std::string decode_gci(const std::string& filename);
-  static std::string decode_dlq(const std::string& filename);
-  static std::pair<std::string, std::string> decode_qst(const std::string& filename);
-
 public:
   enum class FileFormat {
     BIN_DAT = 0,
@@ -60,10 +55,6 @@ public:
   std::u16string short_description;
   std::u16string long_description;
 
-  // these are populated when requested
-  mutable std::shared_ptr<std::string> bin_contents_ptr;
-  mutable std::shared_ptr<std::string> dat_contents_ptr;
-
   Quest(const std::string& file_basename);
   Quest(const Quest&) = default;
   Quest(Quest&&) = default;
@@ -77,13 +68,21 @@ public:
   std::shared_ptr<const std::string> dat_contents() const;
 
   std::shared_ptr<Quest> create_download_quest() const;
+
+  static std::string decode_gci(const std::string& filename);
+  static std::string decode_dlq(const std::string& filename);
+  static std::pair<std::string, std::string> decode_qst(const std::string& filename);
+
+private:
+  // these are populated when requested
+  mutable std::shared_ptr<std::string> bin_contents_ptr;
+  mutable std::shared_ptr<std::string> dat_contents_ptr;
 };
 
 struct QuestIndex {
   std::string directory;
 
   std::map<std::pair<GameVersion, uint64_t>, std::shared_ptr<Quest>> version_menu_item_id_to_quest;
-  std::map<std::pair<GameVersion, std::u16string>, std::shared_ptr<Quest>> version_name_to_quest;
 
   std::map<std::string, std::vector<std::shared_ptr<Quest>>> category_to_quests;
 

src/RareItemSet.hh

@@ -10,6 +10,10 @@ struct RareItemDrop {
 } __attribute__((packed));
 
 struct RareItemSet {
+  // TODO: It looks like this structure can actually vary. We see the offsets
+  // 0194 and 01B2 in the unused section, along with the value 1E (number of box
+  // rares). In PSOGC, these all appear to be the same size/format, but that's
+  // probably not strictly required to be the case.
   // 0x280 in size; describes one difficulty, section ID, and episode
   RareItemDrop rares[0x65];     // 0000 - 0194 in file
   uint8_t box_areas[0x1E];      // 0194 - 01B2 in file

src/ReceiveSubcommands.cc

@@ -163,6 +163,23 @@ static void process_subcommand_word_select(shared_ptr<ServerState>,
   forward_subcommand(l, c, command, flag, data);
 }
 
+// client is done loading into a lobby (we use this to trigger arrow updates)
+static void process_subcommand_set_player_visibility(shared_ptr<ServerState>,
+    shared_ptr<Lobby> l, shared_ptr<Client> c, uint8_t command, uint8_t flag,
+    const string& data) {
+  const auto& p = check_size_sc(data, 4);
+
+  if (p[0].byte[2] != c->lobby_client_id) {
+    return;
+  }
+
+  forward_subcommand(l, c, command, flag, data);
+
+  if (!l->is_game()) {
+    send_arrow_update(l);
+  }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Game commands used by cheat mechanisms
 
@@ -256,12 +273,14 @@ static void process_subcommand_player_drop_item(shared_ptr<ServerState>,
     return;
   }
 
-  l->add_item(c->game_data.player()->remove_item(cmd->item_id, 0),
-      cmd->area, cmd->x, cmd->z);
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    l->add_item(c->game_data.player()->remove_item(cmd->item_id, 0),
+        cmd->area, cmd->x, cmd->z);
 
-  log(INFO, "[Items/%08" PRIX32 "] Player %hhu dropped item %08" PRIX32 " at %hu:(%g, %g)",
-      l->lobby_id, cmd->client_id, cmd->item_id.load(), cmd->area.load(), cmd->x.load(), cmd->z.load());
-  // c->game_data.player()->print_inventory(stderr);
+    log(INFO, "[Items/%08" PRIX32 "] Player %hhu dropped item %08" PRIX32 " at %hu:(%g, %g)",
+        l->lobby_id, cmd->client_id, cmd->item_id.load(), cmd->area.load(), cmd->x.load(), cmd->z.load());
+    c->game_data.player()->print_inventory(stderr);
+  }
 
   forward_subcommand(l, c, command, flag, data);
 }
@@ -280,16 +299,18 @@ static void process_subcommand_create_inventory_item(shared_ptr<ServerState>,
     return;
   }
 
-  PlayerInventoryItem item;
-  item.equip_flags = 0; // TODO: Use the right default flags here
-  item.tech_flag = 0;
-  item.game_flags = 0;
-  item.data = cmd->item;
-  c->game_data.player()->add_item(item);
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    PlayerInventoryItem item;
+    item.equip_flags = 0; // TODO: Use the right default flags here
+    item.tech_flag = 0;
+    item.game_flags = 0;
+    item.data = cmd->item;
+    c->game_data.player()->add_item(item);
 
-  log(INFO, "[Items/%08" PRIX32 "] Player %hhu created inventory item %08" PRIX32,
-      l->lobby_id, cmd->client_id, cmd->item.id.load());
-  // c->game_data.player()->print_inventory(stderr);
+    log(INFO, "[Items/%08" PRIX32 "] Player %hhu created inventory item %08" PRIX32,
+        l->lobby_id, cmd->client_id, cmd->item.id.load());
+    c->game_data.player()->print_inventory(stderr);
+  }
 
   forward_subcommand(l, c, command, flag, data);
 }
@@ -307,18 +328,20 @@ static void process_subcommand_drop_partial_stack(shared_ptr<ServerState>,
     return;
   }
 
-  // TODO: Should we delete anything from the inventory here? Does the client
-  // send an appropriate 6x29 alongside this?
-  PlayerInventoryItem item;
-  item.equip_flags = 0; // TODO: Use the right default flags here
-  item.tech_flag = 0;
-  item.game_flags = 0;
-  item.data = cmd->data;
-  l->add_item(item, cmd->area, cmd->x, cmd->z);
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    // TODO: Should we delete anything from the inventory here? Does the client
+    // send an appropriate 6x29 alongside this?
+    PlayerInventoryItem item;
+    item.equip_flags = 0; // TODO: Use the right default flags here
+    item.tech_flag = 0;
+    item.game_flags = 0;
+    item.data = cmd->data;
+    l->add_item(item, cmd->area, cmd->x, cmd->z);
 
-  log(INFO, "[Items/%08" PRIX32 "] Player %hhu split stack to create ground item %08" PRIX32 " at %hu:(%g, %g)",
-      l->lobby_id, cmd->client_id, item.data.id.load(), cmd->area.load(), cmd->x.load(), cmd->z.load());
-  // c->game_data.player()->print_inventory(stderr);
+    log(INFO, "[Items/%08" PRIX32 "] Player %hhu split stack to create ground item %08" PRIX32 " at %hu:(%g, %g)",
+        l->lobby_id, cmd->client_id, item.data.id.load(), cmd->area.load(), cmd->x.load(), cmd->z.load());
+    c->game_data.player()->print_inventory(stderr);
+  }
 
   forward_subcommand(l, c, command, flag, data);
 }
@@ -333,6 +356,10 @@ static void process_subcommand_drop_partial_stack_bb(shared_ptr<ServerState>,
       return;
     }
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     auto item = c->game_data.player()->remove_item(cmd->item_id, cmd->amount);
 
     // if a stack was split, the original item still exists, so the dropped item
@@ -341,12 +368,17 @@ static void process_subcommand_drop_partial_stack_bb(shared_ptr<ServerState>,
       item.data.id = l->generate_item_id(c->lobby_client_id);
     }
 
+    // PSOBB sends a 6x29 command after it receives the 6x5D, so we need to add
+    // the item back to the player's inventory to correct for this (it will get
+    // removed again by the 6x29 handler)
+    c->game_data.player()->add_item(item);
+
     l->add_item(item, cmd->area, cmd->x, cmd->z);
 
-    log(INFO, "[Items/%08" PRIX32 "] Player %hhu split stack %08" PRIX32 " (%" PRIu32 " of them) at %hu:(%g, %g)",
+    log(INFO, "[Items/%08" PRIX32 "/BB] Player %hhu split stack %08" PRIX32 " (%" PRIu32 " of them) at %hu:(%g, %g)",
         l->lobby_id, cmd->client_id, cmd->item_id.load(), cmd->amount.load(),
         cmd->area.load(), cmd->x.load(), cmd->z.load());
-    // c->game_data.player()->print_inventory(stderr);
+    c->game_data.player()->print_inventory(stderr);
 
     send_drop_stacked_item(l, item.data, cmd->area, cmd->x, cmd->z);
 
@@ -367,16 +399,18 @@ static void process_subcommand_buy_shop_item(shared_ptr<ServerState>,
     return;
   }
 
-  PlayerInventoryItem item;
-  item.equip_flags = 0; // TODO: Use the right default flags here
-  item.tech_flag = 0;
-  item.game_flags = 0;
-  item.data = cmd->item;
-  c->game_data.player()->add_item(item);
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    PlayerInventoryItem item;
+    item.equip_flags = 0; // TODO: Use the right default flags here
+    item.tech_flag = 0;
+    item.game_flags = 0;
+    item.data = cmd->item;
+    c->game_data.player()->add_item(item);
 
-  log(INFO, "[Items/%08" PRIX32 "] Player %hhu bought item %08" PRIX32 " from shop",
-      l->lobby_id, cmd->client_id, item.data.id.load());
-  // c->game_data.player()->print_inventory(stderr);
+    log(INFO, "[Items/%08" PRIX32 "] Player %hhu bought item %08" PRIX32 " from shop",
+        l->lobby_id, cmd->client_id, item.data.id.load());
+    c->game_data.player()->print_inventory(stderr);
+  }
 
   forward_subcommand(l, c, command, flag, data);
 }
@@ -402,7 +436,6 @@ static void process_subcommand_box_or_enemy_item_drop(shared_ptr<ServerState>,
 
   log(INFO, "[Items/%08" PRIX32 "] Leader created ground item %08" PRIX32 " at %hhu:(%g, %g)",
       l->lobby_id, item.data.id.load(), cmd->area, cmd->x.load(), cmd->z.load());
-  // c->game_data.player()->print_inventory(stderr);
 
   forward_subcommand(l, c, command, flag, data);
 }
@@ -413,18 +446,25 @@ static void process_subcommand_pick_up_item(shared_ptr<ServerState>,
     const string& data) {
   auto* cmd = check_size_sc<G_PickUpItem_6x59>(data);
 
-  if (!l->is_game() || (cmd->client_id != c->lobby_client_id)) {
+  if (!l->is_game()) {
     return;
   }
   if (l->version == GameVersion::BB) {
     // BB clients should never send this; only the server should send this
     return;
   }
-  c->game_data.player()->add_item(l->remove_item(cmd->item_id));
 
-  log(INFO, "[Items/%08" PRIX32 "] Player %hu picked up %08" PRIX32,
-      l->lobby_id, cmd->client_id.load(), cmd->item_id.load());
-  // c->game_data.player()->print_inventory(stderr);
+  auto effective_c = l->clients.at(cmd->client_id);
+  if (!effective_c.get()) {
+    return;
+  }
+
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    effective_c->game_data.player()->add_item(l->remove_item(cmd->item_id));
+    log(INFO, "[Items/%08" PRIX32 "] Player %hu picked up %08" PRIX32,
+        l->lobby_id, cmd->client_id.load(), cmd->item_id.load());
+    effective_c->game_data.player()->print_inventory(stderr);
+  }
 
   forward_subcommand(l, c, command, flag, data);
 }
@@ -440,8 +480,16 @@ static void process_subcommand_pick_up_item_request(shared_ptr<ServerState>,
       return;
     }
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     c->game_data.player()->add_item(l->remove_item(cmd->item_id));
 
+    log(INFO, "[Items/%08" PRIX32 "/BB] Player %hhu picked up %08" PRIX32,
+        l->lobby_id, cmd->client_id, cmd->item_id.load());
+    c->game_data.player()->print_inventory(stderr);
+
     send_pick_up_item(l, c, cmd->item_id, cmd->area);
 
   } else {
@@ -452,7 +500,7 @@ static void process_subcommand_pick_up_item_request(shared_ptr<ServerState>,
 static void process_subcommand_equip_unequip_item(shared_ptr<ServerState>,
     shared_ptr<Lobby> l, shared_ptr<Client> c, uint8_t command, uint8_t flag,
     const string& data) {
-  // We don't track equip state on non-BB versions
+  // TODO: We should track equip state on non-BB versions
   if (l->version == GameVersion::BB) {
     const auto* cmd = check_size_sc<G_ItemSubcommand>(data);
 
@@ -460,6 +508,10 @@ static void process_subcommand_equip_unequip_item(shared_ptr<ServerState>,
       return;
     }
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     size_t index = c->game_data.player()->inventory.find_item(cmd->item_id);
     if (cmd->command == 0x25) {
       c->game_data.player()->inventory.items[index].game_flags |= 0x00000008; // equip
@@ -481,12 +533,14 @@ static void process_subcommand_use_item(shared_ptr<ServerState>,
     return;
   }
 
-  size_t index = c->game_data.player()->inventory.find_item(cmd->item_id);
-  player_use_item(c, index);
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    size_t index = c->game_data.player()->inventory.find_item(cmd->item_id);
+    player_use_item(c, index);
 
-  log(INFO, "[Items/%08" PRIX32 "] Player used item %hhu:%08" PRIX32,
-      l->lobby_id, cmd->client_id, cmd->item_id.load());
-  // c->game_data.player()->print_inventory(stderr);
+    log(INFO, "[Items/%08" PRIX32 "] Player used item %hhu:%08" PRIX32,
+        l->lobby_id, cmd->client_id, cmd->item_id.load());
+    c->game_data.player()->print_inventory(stderr);
+  }
 
   forward_subcommand(l, c, command, flag, data);
 }
@@ -542,6 +596,10 @@ static void process_subcommand_bank_action_bb(shared_ptr<ServerState>,
       return;
     }
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     if (cmd->action == 0) { // deposit
       if (cmd->item_id == 0xFFFFFFFF) { // meseta
         if (cmd->meseta_amount > c->game_data.player()->disp.meseta) {
@@ -585,6 +643,10 @@ static void process_subcommand_sort_inventory_bb(shared_ptr<ServerState>,
   if (l->version == GameVersion::BB) {
     const auto* cmd = check_size_sc<G_SortInventory_6xC4>(data);
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     PlayerInventory sorted;
 
     for (size_t x = 0; x < 30; x++) {
@@ -617,6 +679,10 @@ static void process_subcommand_enemy_drop_item_request(shared_ptr<ServerState> s
       return;
     }
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     PlayerInventoryItem item;
 
     // TODO: Deduplicate this code with the box drop item request handler
@@ -671,6 +737,10 @@ static void process_subcommand_box_drop_item_request(shared_ptr<ServerState> s,
       return;
     }
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     PlayerInventoryItem item;
 
     bool is_rare = false;
@@ -861,11 +931,13 @@ static void process_subcommand_destroy_inventory_item(shared_ptr<ServerState>,
   if (cmd->client_id != c->lobby_client_id) {
     return;
   }
-  c->game_data.player()->remove_item(cmd->item_id, cmd->amount);
-  log(INFO, "[Items/%08" PRIX32 "] Inventory item %hhu:%08" PRIX32 " destroyed (%" PRIX32 " of them)",
-      l->lobby_id, cmd->client_id, cmd->item_id.load(), cmd->amount.load());
-  // c->game_data.player()->print_inventory(stderr);
-  forward_subcommand(l, c, command, flag, data);
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    c->game_data.player()->remove_item(cmd->item_id, cmd->amount);
+    log(INFO, "[Items/%08" PRIX32 "] Inventory item %hhu:%08" PRIX32 " destroyed (%" PRIX32 " of them)",
+        l->lobby_id, cmd->client_id, cmd->item_id.load(), cmd->amount.load());
+    c->game_data.player()->print_inventory(stderr);
+    forward_subcommand(l, c, command, flag, data);
+  }
 }
 
 static void process_subcommand_destroy_ground_item(shared_ptr<ServerState>,
@@ -875,11 +947,12 @@ static void process_subcommand_destroy_ground_item(shared_ptr<ServerState>,
   if (!l->is_game()) {
     return;
   }
-  l->remove_item(cmd->item_id);
-  log(INFO, "[Items/%08" PRIX32 "] Ground item %08" PRIX32 " destroyed (%" PRIX32 " of them)",
-      l->lobby_id, cmd->item_id.load(), cmd->amount.load());
-  // c->game_data.player()->print_inventory(stderr);
-  forward_subcommand(l, c, command, flag, data);
+  if (l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED) {
+    l->remove_item(cmd->item_id);
+    log(INFO, "[Items/%08" PRIX32 "] Ground item %08" PRIX32 " destroyed (%" PRIX32 " of them)",
+        l->lobby_id, cmd->item_id.load(), cmd->amount.load());
+    forward_subcommand(l, c, command, flag, data);
+  }
 }
 
 static void process_subcommand_identify_item_bb(shared_ptr<ServerState>,
@@ -891,6 +964,10 @@ static void process_subcommand_identify_item_bb(shared_ptr<ServerState>,
       return;
     }
 
+    if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+      throw logic_error("item tracking not enabled in BB game");
+    }
+
     size_t x = c->game_data.player()->inventory.find_item(cmd->item_id);
     if (c->game_data.player()->inventory.items[x].data.data1[0] != 0) {
       return; // only weapons can be identified
@@ -927,6 +1004,9 @@ static void process_subcommand_identify_item_bb(shared_ptr<ServerState>,
 //     if (cmd->client_id != c->lobby_client_id) {
 //       return;
 //     }
+//     if (!(l->flags & Lobby::Flag::ITEM_TRACKING_ENABLED)) {
+//       throw logic_error("item tracking not enabled in BB game");
+//     }
 //
 //     size_t x = c->game_data.player()->inventory.find_item(cmd->item_id);
 //     c->game_data.player()->inventory.items[x] = c->game_data.player()->identify_result;
@@ -994,6 +1074,16 @@ static void process_subcommand_forward_check_size_ep3_lobby(shared_ptr<ServerSta
   forward_subcommand(l, c, command, flag, data);
 }
 
+static void process_subcommand_forward_check_size_ep3_game(shared_ptr<ServerState>,
+    shared_ptr<Lobby> l, shared_ptr<Client> c, uint8_t command, uint8_t flag,
+    const string& data) {
+  check_size_sc(data, sizeof(PSOSubcommand), 0xFFFF);
+  if (!l->is_game() || !(l->flags & Lobby::Flag::EPISODE_3_ONLY)) {
+    return;
+  }
+  forward_subcommand(l, c, command, flag, data);
+}
+
 static void process_subcommand_invalid(shared_ptr<ServerState>,
     shared_ptr<Lobby>, shared_ptr<Client>, uint8_t command, uint8_t flag,
     const string& data) {
@@ -1061,7 +1151,7 @@ subcommand_handler_t subcommand_handlers[0x100] = {
   /* 20 */ process_subcommand_forward_check_size,
   /* 21 */ process_subcommand_change_area, // Inter-level warp
   /* 22 */ process_subcommand_forward_check_size_client, // Set player visibility
-  /* 23 */ process_subcommand_forward_check_size_client, // Set player visibility
+  /* 23 */ process_subcommand_set_player_visibility, // Set player visibility
   /* 24 */ process_subcommand_forward_check_size_game,
   /* 25 */ process_subcommand_equip_unequip_item, // Equip item
   /* 26 */ process_subcommand_equip_unequip_item, // Unequip item
@@ -1077,7 +1167,7 @@ subcommand_handler_t subcommand_handlers[0x100] = {
   /* 30 */ process_subcommand_forward_check_size_game, // Level up
   /* 31 */ process_subcommand_forward_check_size_game, // Medical center
   /* 32 */ process_subcommand_forward_check_size_game, // Medical center
-  /* 33 */ process_subcommand_forward_check_size_game, // Revive player (only confirmed with moon atomizer)
+  /* 33 */ process_subcommand_forward_check_size_game, // Moon atomizer/Reverser
   /* 34 */ process_subcommand_unimplemented,
   /* 35 */ process_subcommand_unimplemented,
   /* 36 */ process_subcommand_forward_check_game,
@@ -1187,7 +1277,7 @@ subcommand_handler_t subcommand_handlers[0x100] = {
   /* 9E */ process_subcommand_unimplemented,
   /* 9F */ process_subcommand_forward_check_size_game, // Gal Gryphon actions
   /* A0 */ process_subcommand_forward_check_size_game, // Gal Gryphon actions
-  /* A1 */ process_subcommand_unimplemented,
+  /* A1 */ process_subcommand_forward_check_size_game, // Part of revive process. Occurs right after revive command, function unclear.
   /* A2 */ process_subcommand_box_drop_item_request, // Request for item drop from box (handled by server on BB)
   /* A3 */ process_subcommand_forward_check_size_game, // Episode 2 boss actions
   /* A4 */ process_subcommand_forward_check_size_game, // Olga Flow phase 1 actions
@@ -1206,7 +1296,7 @@ subcommand_handler_t subcommand_handlers[0x100] = {
   /* B1 */ process_subcommand_unimplemented,
   /* B2 */ process_subcommand_unimplemented,
   /* B3 */ process_subcommand_unimplemented,
-  /* B4 */ process_subcommand_unimplemented,
+  /* B4 */ process_subcommand_forward_check_size_ep3_game,
   /* B5 */ process_subcommand_open_shop_bb_or_unknown_ep3, // BB shop request
   /* B6 */ process_subcommand_unimplemented, // BB shop contents (server->client only)
   /* B7 */ process_subcommand_unimplemented, // TODO: BB buy shop item

src/SendCommands.cc

@@ -25,110 +25,9 @@ extern FileContentsCache file_cache;
 
 
 
-void send_command(
-    struct bufferevent* bev,
-    GameVersion version,
-    PSOEncryption* crypt,
-    uint16_t command,
-    uint32_t flag,
-    const void* data,
-    size_t size,
-    const char* name_str) {
-  string send_data;
-  size_t logical_size;
-
-  switch (version) {
-    case GameVersion::GC:
-    case GameVersion::DC: {
-      PSOCommandHeaderDCGC header;
-      header.command = command;
-      header.flag = flag;
-      header.size = (sizeof(header) + size + 3) & ~3;
-      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
-      if (size) {
-        send_data.append(reinterpret_cast<const char*>(data), size);
-        send_data.resize(header.size, '\0');
-      }
-      logical_size = header.size;
-      break;
-    }
-
-    case GameVersion::PC:
-    case GameVersion::PATCH: {
-      PSOCommandHeaderPC header;
-      header.size = (sizeof(header) + size + 3) & ~3;
-      header.command = command;
-      header.flag = flag;
-      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
-      if (size) {
-        send_data.append(reinterpret_cast<const char*>(data), size);
-        send_data.resize(header.size, '\0');
-      }
-      logical_size = header.size;
-      break;
-    }
-
-    case GameVersion::BB: {
-      // BB has an annoying behavior here: command lengths must be multiples of
-      // 4, but the actual data length must be a multiple of 8. If the size
-      // field is not divisible by 8, 4 extra bytes are sent anyway.
-      PSOCommandHeaderBB header;
-      header.size = (sizeof(header) + size + 3) & ~3;
-      header.command = command;
-      header.flag = flag;
-      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
-      if (size) {
-        send_data.append(reinterpret_cast<const char*>(data), size);
-        if (crypt) {
-          send_data.resize((send_data.size() + 7) & ~7, '\0');
-        } else {
-          send_data.resize(header.size, '\0');
-        }
-      }
-      logical_size = header.size;
-      break;
-    }
-
-    default:
-      throw logic_error("unimplemented game version in send_command");
-  }
-
-  if (name_str) {
-    string name_token;
-    if (name_str[0]) {
-      name_token = string(" to ") + name_str;
-    }
-    if (use_terminal_colors) {
-      print_color_escape(stderr, TerminalFormat::FG_YELLOW, TerminalFormat::BOLD, TerminalFormat::END);
-    }
-    log(INFO, "Sending%s (version=%s command=%04hX flag=%08X)",
-        name_token.c_str(), name_for_version(version), command, flag);
-    print_data(stderr, send_data.data(), logical_size);
-    if (use_terminal_colors) {
-      print_color_escape(stderr, TerminalFormat::NORMAL, TerminalFormat::END);
-    }
-  }
-
-  if (crypt) {
-    crypt->encrypt(send_data.data(), send_data.size());
-  }
-
-  struct evbuffer* buf = bufferevent_get_output(bev);
-  evbuffer_add(buf, send_data.data(), send_data.size());
-}
-
 void send_command(shared_ptr<Client> c, uint16_t command, uint32_t flag,
     const void* data, size_t size) {
-  if (!c->bev) {
-    return;
-  }
-  string encoded_name;
-  auto player = c->game_data.player(false);
-  if (player) {
-    encoded_name = remove_language_marker(encode_sjis(player->disp.name));
-  }
-  send_command(c->bev, c->version, c->crypt_out.get(), command, flag, data,
-      size, encoded_name.c_str());
+  c->channel.send(command, flag, data, size);
 }
 
 void send_command_excluding_client(shared_ptr<Lobby> l, shared_ptr<Client> c,
@@ -153,6 +52,32 @@ void send_command(shared_ptr<ServerState> s, uint16_t command, uint32_t flag,
   }
 }
 
+template <typename HeaderT>
+void send_command_with_header_t(shared_ptr<Client> c, const void* data,
+    size_t size) {
+  const HeaderT* header = reinterpret_cast<const HeaderT*>(data);
+  send_command(c, header->command, header->flag, header + 1, size - sizeof(HeaderT));
+}
+
+void send_command_with_header(shared_ptr<Client> c, const void* data,
+    size_t size) {
+  switch (c->version) {
+    case GameVersion::GC:
+    case GameVersion::DC:
+      send_command_with_header_t<PSOCommandHeaderDCGC>(c, data, size);
+      break;
+    case GameVersion::PC:
+    case GameVersion::PATCH:
+      send_command_with_header_t<PSOCommandHeaderPC>(c, data, size);
+      break;
+    case GameVersion::BB:
+      send_command_with_header_t<PSOCommandHeaderBB>(c, data, size);
+      break;
+    default:
+      throw logic_error("unimplemented game version in send_command_with_header");
+  }
+}
+
 
 
 // specific command sending functions follow. in general, they're written in
@@ -172,13 +97,14 @@ static const char* anti_copyright = "This server is in no way affiliated, sponso
 static const char* dc_port_map_copyright = "DreamCast Port Map. Copyright SEGA Enterprises. 1999";
 static const char* dc_lobby_server_copyright = "DreamCast Lobby Server. Copyright SEGA Enterprises. 1999";
 static const char* bb_game_server_copyright = "Phantasy Star Online Blue Burst Game Server. Copyright 1999-2004 SONICTEAM.";
+static const char* bb_pm_server_copyright = "PSO NEW PM Server. Copyright 1999-2002 SONICTEAM.";
 static const char* patch_server_copyright = "Patch Server. Copyright SonicTeam, LTD. 2001";
 
-S_ServerInit_DC_PC_GC_02_17 prepare_server_init_contents_dc_pc_gc(
+S_ServerInit_DC_PC_GC_02_17_92_9B prepare_server_init_contents_dc_pc_gc(
     bool initial_connection,
     uint32_t server_key,
     uint32_t client_key) {
-  S_ServerInit_DC_PC_GC_02_17 cmd;
+  S_ServerInit_DC_PC_GC_02_17_92_9B cmd;
   cmd.copyright = initial_connection
       ? dc_port_map_copyright : dc_lobby_server_copyright;
   cmd.server_key = server_key;
@@ -200,12 +126,12 @@ void send_server_init_dc_pc_gc(shared_ptr<Client> c,
   switch (c->version) {
     case GameVersion::DC:
     case GameVersion::PC:
-      c->crypt_out.reset(new PSOPCEncryption(server_key));
-      c->crypt_in.reset(new PSOPCEncryption(client_key));
+      c->channel.crypt_out.reset(new PSOPCEncryption(server_key));
+      c->channel.crypt_in.reset(new PSOPCEncryption(client_key));
       break;
     case GameVersion::GC:
-      c->crypt_out.reset(new PSOGCEncryption(server_key));
-      c->crypt_in.reset(new PSOGCEncryption(client_key));
+      c->channel.crypt_out.reset(new PSOGCEncryption(server_key));
+      c->channel.crypt_in.reset(new PSOGCEncryption(client_key));
       break;
     default:
       throw invalid_argument("incorrect client version");
@@ -214,28 +140,34 @@ void send_server_init_dc_pc_gc(shared_ptr<Client> c,
 
 S_ServerInit_BB_03 prepare_server_init_contents_bb(
     const parray<uint8_t, 0x30>& server_key,
-    const parray<uint8_t, 0x30>& client_key) {
+    const parray<uint8_t, 0x30>& client_key,
+    bool use_secondary_message) {
   S_ServerInit_BB_03 cmd;
-  cmd.copyright = bb_game_server_copyright;
+  cmd.copyright = use_secondary_message ? bb_pm_server_copyright : bb_game_server_copyright;
   cmd.server_key = server_key;
   cmd.client_key = client_key;
   cmd.after_message = anti_copyright;
   return cmd;
 }
 
-void send_server_init_bb(shared_ptr<ServerState> s, shared_ptr<Client> c) {
+void send_server_init_bb(shared_ptr<ServerState> s, shared_ptr<Client> c,
+    bool use_secondary_message) {
   parray<uint8_t, 0x30> server_key;
   parray<uint8_t, 0x30> client_key;
   random_data(server_key.data(), server_key.bytes());
   random_data(client_key.data(), client_key.bytes());
-  auto cmd = prepare_server_init_contents_bb(server_key, client_key);
-  send_command_t(c, 0x03, 0x00, cmd);
+  auto cmd = prepare_server_init_contents_bb(server_key, client_key, use_secondary_message);
+  send_command_t(c, use_secondary_message ? 0x9B : 0x03, 0x00, cmd);
 
-  static const string expected_first_data("\xB4\x00\x93\x00\x00\x00\x00\x00", 8);
+  static const string primary_expected_first_data("\xB4\x00\x93\x00\x00\x00\x00\x00", 8);
+  static const string secondary_expected_first_data("\xDC\x00\xDB\x00\x00\x00\x00\x00", 8);
   shared_ptr<PSOBBMultiKeyDetectorEncryption> detector_crypt(new PSOBBMultiKeyDetectorEncryption(
-      s->bb_private_keys, expected_first_data, cmd.client_key.data(), sizeof(cmd.client_key)));
-  c->crypt_in = detector_crypt;
-  c->crypt_out.reset(new PSOBBMultiKeyImitatorEncryption(
+      s->bb_private_keys,
+      use_secondary_message ? secondary_expected_first_data : primary_expected_first_data,
+      cmd.client_key.data(),
+      sizeof(cmd.client_key)));
+  c->channel.crypt_in = detector_crypt;
+  c->channel.crypt_out.reset(new PSOBBMultiKeyImitatorEncryption(
       detector_crypt, cmd.server_key.data(), sizeof(cmd.server_key), true));
 }
 
@@ -249,12 +181,12 @@ void send_server_init_patch(shared_ptr<Client> c) {
   cmd.client_key = client_key;
   send_command_t(c, 0x02, 0x00, cmd);
 
-  c->crypt_out.reset(new PSOPCEncryption(server_key));
-  c->crypt_in.reset(new PSOPCEncryption(client_key));
+  c->channel.crypt_out.reset(new PSOPCEncryption(server_key));
+  c->channel.crypt_in.reset(new PSOPCEncryption(client_key));
 }
 
 void send_server_init(shared_ptr<ServerState> s, shared_ptr<Client> c,
-    bool initial_connection) {
+    bool initial_connection, bool use_secondary_message) {
   switch (c->version) {
     case GameVersion::DC:
     case GameVersion::PC:
@@ -265,7 +197,7 @@ void send_server_init(shared_ptr<ServerState> s, shared_ptr<Client> c,
       send_server_init_patch(c);
       break;
     case GameVersion::BB:
-      send_server_init_bb(s, c);
+      send_server_init_bb(s, c, use_secondary_message);
       break;
     default:
       throw logic_error("unimplemented versioned command");
@@ -285,6 +217,38 @@ void send_update_client_config(shared_ptr<Client> c) {
 
 
 
+void send_function_call(
+    shared_ptr<Client> c,
+    shared_ptr<CompiledFunctionCode> code,
+    const std::unordered_map<std::string, uint32_t>& label_writes,
+    const std::string& suffix,
+    uint32_t checksum_addr,
+    uint32_t checksum_size) {
+  if (c->version != GameVersion::GC) {
+    throw logic_error("cannot send function calls to non-GameCube clients");
+  }
+  if (c->flags & Client::Flag::EPISODE_3) {
+    throw logic_error("cannot send function calls to Episode 3 clients");
+  }
+
+  string data;
+  uint32_t index = 0;
+  if (code.get()) {
+    data = code->generate_client_command(label_writes, suffix);
+    index = code->index;
+  }
+
+  S_ExecuteCode_B2 header = {data.size(), checksum_addr, checksum_size};
+
+  StringWriter w;
+  w.put(header);
+  w.write(data);
+
+  send_command(c, 0xB2, index, w.str());
+}
+
+
+
 void send_reconnect(shared_ptr<Client> c, uint32_t address, uint16_t port) {
   S_Reconnect_19 cmd = {address, port, 0};
   // On the patch server, 14 is the reconnect command, but it works exactly the
@@ -435,7 +399,7 @@ void send_stream_file_chunk_bb(shared_ptr<Client> c, uint32_t chunk_index) {
 }
 
 void send_approve_player_choice_bb(shared_ptr<Client> c) {
-  S_ApprovePlayerChoice_BB_00E4 cmd = {c->bb_player_index, 1};
+  S_ApprovePlayerChoice_BB_00E4 cmd = {c->game_data.bb_player_index, 1};
   send_command_t(c, 0x00E4, 0x00000000, cmd);
 }
 
@@ -458,56 +422,70 @@ void send_enter_directory_patch(shared_ptr<Client> c, const string& dir) {
 ////////////////////////////////////////////////////////////////////////////////
 // message functions
 
-void send_text(shared_ptr<Client> c, StringWriter& w, uint16_t command,
-    const u16string& text) {
-  if ((c->version == GameVersion::DC) || (c->version == GameVersion::GC)) {
+void send_text(Channel& ch, StringWriter& w, uint16_t command,
+    const u16string& text, bool should_add_color) {
+  if ((ch.version == GameVersion::DC) || (ch.version == GameVersion::GC)) {
     string data = encode_sjis(text);
-    add_color(w, data.c_str(), data.size());
+    if (should_add_color) {
+      add_color(w, data.c_str(), data.size());
+    } else {
+      w.write(data);
+    }
+    w.put_u8(0);
   } else {
-    add_color(w, text.c_str(), text.size());
+    if (should_add_color) {
+      add_color(w, text.c_str(), text.size());
+    } else {
+      w.write(text.data(), text.size() * sizeof(char16_t));
+    }
+    w.put_u16(0);
   }
   while (w.str().size() & 3) {
     w.put_u8(0);
   }
-  send_command(c, command, 0x00, w.str());
+  ch.send(command, 0x00, w.str());
 }
 
-void send_header_text(shared_ptr<Client> c, uint16_t command,
-    uint32_t guild_card_number, const u16string& text) {
+void send_text(Channel& ch, uint16_t command, const u16string& text, bool should_add_color) {
   StringWriter w;
-  w.put(SC_TextHeader_01_06_11_B0({0, guild_card_number}));
-  send_text(c, w, command, text);
+  send_text(ch, w, command, text, should_add_color);
 }
 
-void send_text(shared_ptr<Client> c, uint16_t command,
-    const u16string& text) {
+void send_header_text(Channel& ch, uint16_t command,
+    uint32_t guild_card_number, const u16string& text, bool should_add_color) {
   StringWriter w;
-  send_text(c, w, command, text);
+  w.put(SC_TextHeader_01_06_11_B0_EE({0, guild_card_number}));
+  send_text(ch, w, command, text, should_add_color);
 }
 
 void send_message_box(shared_ptr<Client> c, const u16string& text) {
   uint16_t command = (c->version == GameVersion::PATCH) ? 0x13 : 0x1A;
-  send_text(c, command, text);
+  send_text(c->channel, command, text, true);
 }
 
 void send_lobby_name(shared_ptr<Client> c, const u16string& text) {
-  send_text(c, 0x8A, text);
+  send_text(c->channel, 0x8A, text, false);
 }
 
-void send_quest_info(shared_ptr<Client> c, const u16string& text) {
-  send_text(c, 0xA3, text);
+void send_quest_info(shared_ptr<Client> c, const u16string& text,
+    bool is_download_quest) {
+  send_text(c->channel, is_download_quest ? 0xA5 : 0xA3, text, true);
 }
 
 void send_lobby_message_box(shared_ptr<Client> c, const u16string& text) {
-  send_header_text(c, 0x01, 0, text);
+  send_header_text(c->channel, 0x01, 0, text, true);
 }
 
 void send_ship_info(shared_ptr<Client> c, const u16string& text) {
-  send_header_text(c, 0x11, 0, text);
+  send_header_text(c->channel, 0x11, 0, text, true);
+}
+
+void send_text_message(Channel& ch, const std::u16string& text) {
+  send_header_text(ch, 0xB0, 0, text, true);
 }
 
 void send_text_message(shared_ptr<Client> c, const u16string& text) {
-  send_header_text(c, 0xB0, 0, text);
+  send_header_text(c->channel, 0xB0, 0, text, true);
 }
 
 void send_text_message(shared_ptr<Lobby> l, const u16string& text) {
@@ -526,6 +504,10 @@ void send_text_message(shared_ptr<ServerState> s, const u16string& text) {
   }
 }
 
+void send_chat_message(Channel& ch, const u16string& text) {
+  send_header_text(ch, 0x06, 0, text, false);
+}
+
 void send_chat_message(shared_ptr<Client> c, uint32_t from_guild_card_number,
     const u16string& from_name, const u16string& text) {
   u16string data;
@@ -535,10 +517,10 @@ void send_chat_message(shared_ptr<Client> c, uint32_t from_guild_card_number,
   data.append(remove_language_marker(from_name));
   data.append(u"\x09\x09J");
   data.append(text);
-  send_header_text(c, 0x06, from_guild_card_number, data);
+  send_header_text(c->channel, 0x06, from_guild_card_number, data, false);
 }
 
-void send_simple_mail_gc(std::shared_ptr<Client> c, uint32_t from_guild_card_number,
+void send_simple_mail_gc(shared_ptr<Client> c, uint32_t from_guild_card_number,
     const u16string& from_name, const u16string& text) {
   SC_SimpleMail_GC_81 cmd;
   cmd.player_tag = 0x00010000;
@@ -549,7 +531,7 @@ void send_simple_mail_gc(std::shared_ptr<Client> c, uint32_t from_guild_card_num
   send_command_t(c, 0x81, 0x00, cmd);
 }
 
-void send_simple_mail(std::shared_ptr<Client> c, uint32_t from_guild_card_number,
+void send_simple_mail(shared_ptr<Client> c, uint32_t from_guild_card_number,
     const u16string& from_name, const u16string& text) {
   if (c->version == GameVersion::GC) {
     send_simple_mail_gc(c, from_guild_card_number, from_name, text);
@@ -608,7 +590,8 @@ void send_card_search_result_t(
   // TODO: make this actually make sense... currently we just take the sockname
   // for the target client. This also doesn't work if the client is on a virtual
   // connection (the address and port are zero).
-  const sockaddr_in* local_addr = reinterpret_cast<const sockaddr_in*>(&result->local_addr);
+  const sockaddr_in* local_addr = reinterpret_cast<const sockaddr_in*>(
+      &result->channel.local_addr);
   cmd.reconnect_command.address = local_addr->sin_addr.s_addr;
   cmd.reconnect_command.port = ntohs(local_addr->sin_port);
   cmd.reconnect_command.unused = 0;
@@ -623,7 +606,7 @@ void send_card_search_result_t(
     location_string = string_printf(",BLOCK00,%s", encoded_server_name.c_str());
   }
   cmd.location_string = location_string;
-  cmd.menu_id = LOBBY_MENU_ID;
+  cmd.menu_id = MenuID::LOBBY;
   cmd.lobby_id = result->lobby_id;
   cmd.name = result->game_data.player()->disp.name;
 
@@ -724,7 +707,9 @@ void send_menu_t(
         ((c->version == GameVersion::PC) && (item.flags & MenuItem::Flag::INVISIBLE_ON_PC)) ||
         ((c->version == GameVersion::GC) && (item.flags & MenuItem::Flag::INVISIBLE_ON_GC)) ||
         ((c->version == GameVersion::BB) && (item.flags & MenuItem::Flag::INVISIBLE_ON_BB)) ||
-        ((item.flags & MenuItem::Flag::REQUIRES_MESSAGE_BOXES) && (c->flags & Client::Flag::NO_MESSAGE_BOX_CLOSE_CONFIRMATION))) {
+        ((item.flags & MenuItem::Flag::REQUIRES_MESSAGE_BOXES) && (c->flags & Client::Flag::NO_MESSAGE_BOX_CLOSE_CONFIRMATION)) ||
+        ((item.flags & MenuItem::Flag::REQUIRES_SEND_FUNCTION_CALL) && (c->flags & Client::Flag::DOES_NOT_SUPPORT_SEND_FUNCTION_CALL)) ||
+        ((item.flags & MenuItem::Flag::REQUIRES_SAVE_DISABLED) && (c->flags & Client::Flag::SAVE_ENABLED))) {
       continue;
     }
     auto& e = entries.emplace_back();
@@ -755,7 +740,7 @@ void send_game_menu_t(shared_ptr<Client> c, shared_ptr<ServerState> s) {
   vector<S_GameMenuEntry<CharT>> entries;
   {
     auto& e = entries.emplace_back();
-    e.menu_id = GAME_MENU_ID;
+    e.menu_id = MenuID::GAME;
     e.game_id = 0x00000000;
     e.difficulty_tag = 0x00;
     e.num_players = 0x00;
@@ -774,7 +759,7 @@ void send_game_menu_t(shared_ptr<Client> c, shared_ptr<ServerState> s) {
     }
 
     auto& e = entries.emplace_back();
-    e.menu_id = GAME_MENU_ID;
+    e.menu_id = MenuID::GAME;
     e.game_id = l->lobby_id;
     e.difficulty_tag = (l_is_ep3 ? 0x0A : (l->difficulty + 0x22));
     e.num_players = l->count_clients();
@@ -850,7 +835,7 @@ void send_quest_menu(shared_ptr<Client> c, uint32_t menu_id,
 }
 
 void send_quest_menu(shared_ptr<Client> c, uint32_t menu_id,
-    const std::vector<MenuItem>& items, bool is_download_menu) {
+    const vector<MenuItem>& items, bool is_download_menu) {
   if (c->version == GameVersion::PC) {
     send_quest_menu_t<S_QuestMenuEntry_PC_A2_A4>(c, menu_id, items, is_download_menu);
   } else if (c->version == GameVersion::GC) {
@@ -876,7 +861,7 @@ void send_lobby_list(shared_ptr<Client> c, shared_ptr<ServerState> s) {
       continue;
     }
     auto& e = entries.emplace_back();
-    e.menu_id = LOBBY_MENU_ID;
+    e.menu_id = MenuID::LOBBY;
     e.item_id = l->lobby_id;
     e.unused = 0;
   }
@@ -1054,10 +1039,15 @@ void send_player_join_notification(shared_ptr<Client> c,
 }
 
 void send_player_leave_notification(shared_ptr<Lobby> l, uint8_t leaving_client_id) {
-  S_LeaveLobby_66_69 cmd = {leaving_client_id, l->leader_id, 0};
+  S_LeaveLobby_66_69_Ep3_E9 cmd = {leaving_client_id, l->leader_id, 0};
   send_command_t(l, l->is_game() ? 0x66 : 0x69, leaving_client_id, cmd);
 }
 
+void send_self_leave_notification(shared_ptr<Client> c) {
+  S_LeaveLobby_66_69_Ep3_E9 cmd = {c->lobby_client_id, 0, 0};
+  send_command_t(c, 0x69, c->lobby_client_id, cmd);
+}
+
 void send_get_player_info(shared_ptr<Client> c) {
   send_command(c, 0x95, 0x00);
 }
@@ -1124,6 +1114,16 @@ void send_player_stats_change(shared_ptr<Lobby> l, shared_ptr<Client> c,
   send_command_vt(l, 0x60, 0x00, subs);
 }
 
+void send_warp(Channel& ch, uint8_t client_id, uint32_t area) {
+  PSOSubcommand cmds[2];
+  cmds[0].byte[0] = 0x94;
+  cmds[0].byte[1] = 0x02;
+  cmds[0].byte[2] = client_id;
+  cmds[0].byte[3] = 0x00;
+  cmds[1].dword = area;
+  ch.send(0x62, client_id, cmds, 8);
+}
+
 void send_warp(shared_ptr<Client> c, uint32_t area) {
   PSOSubcommand cmds[2];
   cmds[0].byte[0] = 0x94;
@@ -1169,25 +1169,24 @@ void send_revive_player(shared_ptr<Lobby> l, shared_ptr<Client> c) {
 ////////////////////////////////////////////////////////////////////////////////
 // BB game commands
 
-// notifies other players of a dropped item from a box or enemy
 void send_drop_item(shared_ptr<Lobby> l, const ItemData& item,
     bool from_enemy, uint8_t area, float x, float z, uint16_t request_id) {
   G_DropItem_6x5F cmd = {
-      0x5F, 0x0A, 0x0000, area, from_enemy, request_id, x, z, 0, item, 0};
+      0x5F, 0x0B, 0x0000, area, from_enemy, request_id, x, z, 0, item, 0};
   send_command_t(l, 0x60, 0x00, cmd);
 }
 
-// notifies other players that a stack was split and part of it dropped (a new item was created)
+// Notifies other players that a stack was split and part of it dropped (a new
+// item was created)
 void send_drop_stacked_item(shared_ptr<Lobby> l, const ItemData& item,
     uint8_t area, float x, float z) {
   // TODO: Is this order correct? The original code sent {item, 0}, but it seems
   // GC sends {0, item} (the last two fields in the struct are switched).
   G_DropStackedItem_6x5D cmd = {
-      0x5D, 0x09, 0x00, 0x00, area, 0, x, z, item, 0};
+      0x5D, 0x0A, 0x00, 0x00, area, 0, x, z, item, 0};
   send_command_t(l, 0x60, 0x00, cmd);
 }
 
-// notifies other players that an item was picked up
 void send_pick_up_item(shared_ptr<Lobby> l, shared_ptr<Client> c,
     uint32_t item_id, uint8_t area) {
   G_PickUpItem_6x59 cmd = {
@@ -1195,7 +1194,8 @@ void send_pick_up_item(shared_ptr<Lobby> l, shared_ptr<Client> c,
   send_command_t(l, 0x60, 0x00, cmd);
 }
 
-// creates an item in a player's inventory (used for withdrawing items from the bank)
+// Creates an item in a player's inventory (used for withdrawing items from the
+// bank)
 void send_create_inventory_item(shared_ptr<Lobby> l, shared_ptr<Client> c,
     const ItemData& item) {
   G_CreateInventoryItem_BB_6xBE cmd = {
@@ -1292,13 +1292,12 @@ void send_give_experience(shared_ptr<Lobby> l, shared_ptr<Client> c,
 ////////////////////////////////////////////////////////////////////////////////
 // ep3 only commands
 
-// sends the (PRS-compressed) card list to the client
-void send_ep3_card_list_update(shared_ptr<Client> c) {
-  auto file_data = file_cache.get("system/ep3/cardupdate.mnr");
+void send_ep3_card_list_update(shared_ptr<ServerState> s, shared_ptr<Client> c) {
+  const auto& data = s->ep3_data_index->get_compressed_card_definitions();
 
   StringWriter w;
-  w.put_u32l(file_data->size());
-  w.write(*file_data);
+  w.put_u32l(data.size());
+  w.write(data);
 
   send_command(c, 0xB8, 0x00, w.str());
 }
@@ -1311,31 +1310,36 @@ void send_ep3_rank_update(shared_ptr<Client> c) {
 }
 
 // sends the map list (used for battle setup) to all players in a game
-void send_ep3_map_list(shared_ptr<Lobby> l) {
-  auto file_data = file_cache.get("system/ep3/maplist.mnr");
+void send_ep3_map_list(shared_ptr<ServerState> s, shared_ptr<Lobby> l) {
+  const auto& data = s->ep3_data_index->get_compressed_map_list();
 
-  string data(16, '\0');
-  PSOSubcommand* subs = reinterpret_cast<PSOSubcommand*>(data.data());
+  string cmd_data(16, '\0');
+  PSOSubcommand* subs = reinterpret_cast<PSOSubcommand*>(cmd_data.data());
   subs[0].dword = 0x000000B6;
-  subs[1].dword = (23 + file_data->size()) & 0xFFFFFFFC;
+  subs[1].dword = (data.size() + 0x14 + 3) & 0xFFFFFFFC;
   subs[2].dword = 0x00000040;
-  subs[3].dword = file_data->size();
-  data += *file_data;
+  subs[3].dword = data.size();
+  cmd_data += data;
 
-  send_command(l, 0x6C, 0x00, data);
+  send_command(l, 0x6C, 0x00, cmd_data);
 }
 
 // sends the map data for the chosen map to all players in the game
-void send_ep3_map_data(shared_ptr<Lobby> l, uint32_t map_id) {
-  string filename = string_printf("system/ep3/map%08" PRIX32 ".mnm", map_id);
-  auto file_data = file_cache.get(filename);
+void send_ep3_map_data(shared_ptr<ServerState> s, shared_ptr<Lobby> l, uint32_t map_id) {
+  auto entry = s->ep3_data_index->get_map(map_id);
+  const auto& compressed = entry->compressed();
 
-  string data(12, '\0');
+  string data(0x14, '\0');
   PSOSubcommand* subs = reinterpret_cast<PSOSubcommand*>(data.data());
   subs[0].dword = 0x000000B6;
-  subs[1].dword = (19 + file_data->size()) & 0xFFFFFFFC;
+  subs[1].dword = (19 + compressed.size()) & 0xFFFFFFFC;
   subs[2].dword = 0x00000041;
-  data += *file_data;
+  subs[3].dword = entry->map.map_number.load();
+  subs[4].dword = compressed.size();
+  data += compressed;
+  while (data.size() & 3) {
+    data.push_back('\0');
+  }
 
   send_command(l, 0x6C, 0x00, data);
 }
@@ -1375,7 +1379,7 @@ void send_quest_open_file_t(
     default:
       throw logic_error("invalid quest file type");
   }
-  cmd.unused = 0;
+  cmd.unused.clear();
   cmd.file_size = file_size;
   cmd.filename = filename.c_str();
   send_command_t(c, command_num, 0x00, cmd);

src/SendCommands.hh

@@ -14,20 +14,23 @@
 #include "Quest.hh"
 #include "Text.hh"
 #include "CommandFormats.hh"
+#include "FunctionCompiler.hh"
 
 
 
+// TODO: Many of these functions should take a Channel& instead of a
+// shared_ptr<Client>. Refactor functions appropriately.
+
 // Note: There are so many versions of this function for a few reasons:
 // - There are a lot of different target types (sometimes we want to send a
 //   command to one client, sometimes to everyone in a lobby, etc.)
 // - For the const void* versions, the data and size arguments should not be
 //   independently optional - this can lead to bugs where a non-null data
-//   pointer is given but size is accidentally not given zero (e.g. if the type
-//   of data in the calling function is changed from string to void*).
+//   pointer is given but size is accidentally not given (e.g. if the type of
+//   data in the calling function is changed from string to void*).
 
-void send_command(struct bufferevent* bev, GameVersion version,
-    PSOEncryption* crypt, uint16_t command, uint32_t flag, const void* data,
-    size_t size, const char* name_str = nullptr);
+void send_command(Channel& ch, uint16_t command, uint32_t flag,
+    const void* data, size_t size);
 
 void send_command(std::shared_ptr<Client> c, uint16_t command,
     uint32_t flag, const void* data, size_t size);
@@ -88,19 +91,31 @@ void send_command_t_vt(std::shared_ptr<TargetT> c, uint16_t command,
   send_command(c, command, flag, all_data.data(), all_data.size());
 }
 
+void send_command_with_header(std::shared_ptr<Client> c, const void* data,
+    size_t size);
 
 
-S_ServerInit_DC_PC_GC_02_17 prepare_server_init_contents_dc_pc_gc(
+
+S_ServerInit_DC_PC_GC_02_17_92_9B prepare_server_init_contents_dc_pc_gc(
     bool initial_connection,
     uint32_t server_key,
     uint32_t client_key);
 S_ServerInit_BB_03 prepare_server_init_contents_bb(
     const parray<uint8_t, 0x30>& server_key,
-    const parray<uint8_t, 0x30>& client_key);
+    const parray<uint8_t, 0x30>& client_key,
+    bool use_secondary_message);
 void send_server_init(std::shared_ptr<ServerState> s, std::shared_ptr<Client> c,
-    bool initial_connection);
+    bool initial_connection, bool use_secondary_message);
 void send_update_client_config(std::shared_ptr<Client> c);
 
+void send_function_call(
+    std::shared_ptr<Client> c,
+    std::shared_ptr<CompiledFunctionCode> code,
+    const std::unordered_map<std::string, uint32_t>& label_writes = {},
+    const std::string& suffix = "",
+    uint32_t checksum_addr = 0,
+    uint32_t checksum_size = 0);
+
 void send_reconnect(std::shared_ptr<Client> c, uint32_t address, uint16_t port);
 void send_pc_gc_split_reconnect(std::shared_ptr<Client> c, uint32_t address,
     uint16_t pc_port, uint16_t gc_port);
@@ -121,12 +136,15 @@ void send_enter_directory_patch(std::shared_ptr<Client> c, const std::string& di
 
 void send_message_box(std::shared_ptr<Client> c, const std::u16string& text);
 void send_lobby_name(std::shared_ptr<Client> c, const std::u16string& text);
-void send_quest_info(std::shared_ptr<Client> c, const std::u16string& text);
+void send_quest_info(std::shared_ptr<Client> c, const std::u16string& text,
+    bool is_download_quest);
 void send_lobby_message_box(std::shared_ptr<Client> c, const std::u16string& text);
 void send_ship_info(std::shared_ptr<Client> c, const std::u16string& text);
+void send_text_message(Channel& ch, const std::u16string& text);
 void send_text_message(std::shared_ptr<Client> c, const std::u16string& text);
 void send_text_message(std::shared_ptr<Lobby> l, const std::u16string& text);
 void send_text_message(std::shared_ptr<ServerState> l, const std::u16string& text);
+void send_chat_message(Channel& ch, const std::u16string& text);
 void send_chat_message(std::shared_ptr<Client> c, uint32_t from_serial_number,
     const std::u16string& from_name, const std::u16string& text);
 void send_simple_mail(std::shared_ptr<Client> c, uint32_t from_serial_number,
@@ -134,7 +152,7 @@ void send_simple_mail(std::shared_ptr<Client> c, uint32_t from_serial_number,
 
 template <typename TargetT>
 __attribute__((format(printf, 2, 3))) void send_text_message_printf(
-    std::shared_ptr<TargetT> t, const char* format, ...) {
+    TargetT& t, const char* format, ...) {
   va_list va;
   va_start(va, format);
   std::string buf = string_vprintf(format, va);
@@ -153,7 +171,7 @@ void send_card_search_result(
 
 void send_guild_card(std::shared_ptr<Client> c, std::shared_ptr<Client> source);
 void send_menu(std::shared_ptr<Client> c, const std::u16string& menu_name,
-    uint32_t menu_id, const std::vector<MenuItem>& items, bool is_info_menu);
+    uint32_t menu_id, const std::vector<MenuItem>& items, bool is_info_menu = false);
 void send_game_menu(std::shared_ptr<Client> c, std::shared_ptr<ServerState> s);
 void send_quest_menu(std::shared_ptr<Client> c, uint32_t menu_id,
     const std::vector<std::shared_ptr<const Quest>>& quests, bool is_download_menu);
@@ -166,6 +184,7 @@ void send_player_join_notification(std::shared_ptr<Client> c,
     std::shared_ptr<Lobby> l, std::shared_ptr<Client> joining_client);
 void send_player_leave_notification(std::shared_ptr<Lobby> l,
     uint8_t leaving_client_id);
+void send_self_leave_notification(std::shared_ptr<Client> c);
 void send_get_player_info(std::shared_ptr<Client> c);
 
 void send_arrow_update(std::shared_ptr<Lobby> l);
@@ -182,6 +201,7 @@ enum PlayerStatsChange {
 
 void send_player_stats_change(std::shared_ptr<Lobby> l, std::shared_ptr<Client> c,
     PlayerStatsChange which, uint32_t amount);
+void send_warp(Channel& ch, uint8_t client_id, uint32_t area);
 void send_warp(std::shared_ptr<Client> c, uint32_t area);
 
 void send_ep3_change_music(std::shared_ptr<Client> c, uint32_t song);
@@ -204,10 +224,13 @@ void send_shop(std::shared_ptr<Client> c, uint8_t shop_type);
 void send_level_up(std::shared_ptr<Lobby> l, std::shared_ptr<Client> c);
 void send_give_experience(std::shared_ptr<Lobby> l, std::shared_ptr<Client> c,
     uint32_t amount);
-void send_ep3_card_list_update(std::shared_ptr<Client> c);
+void send_ep3_card_list_update(
+    std::shared_ptr<ServerState> s, std::shared_ptr<Client> c);
 void send_ep3_rank_update(std::shared_ptr<Client> c);
-void send_ep3_map_list(std::shared_ptr<Lobby> l);
-void send_ep3_map_data(std::shared_ptr<Lobby> l, uint32_t map_id);
+void send_ep3_map_list(
+    std::shared_ptr<ServerState> s, std::shared_ptr<Lobby> l);
+void send_ep3_map_data(
+    std::shared_ptr<ServerState> s, std::shared_ptr<Lobby> l, uint32_t map_id);
 
 enum class QuestFileType {
   ONLINE = 0,

src/Server.cc

@@ -24,41 +24,29 @@
 #include "ReceiveCommands.hh"
 
 using namespace std;
+using namespace std::placeholders;
 
 
 
-void Server::disconnect_client(struct bufferevent* bev) {
-  this->disconnect_client(this->bev_to_client.at(bev));
-}
-
 void Server::disconnect_client(shared_ptr<Client> c) {
-  this->bev_to_client.erase(c->bev);
-  struct bufferevent* bev = c->bev;
-  c->bev = nullptr;
-
-  int fd = bufferevent_getfd(bev);
-  if (fd < 0) {
-    this->log(INFO, "Client on virtual connection %p disconnected", bev);
+  if (c->channel.is_virtual_connection) {
+    this->log(INFO, "Disconnecting client on virtual connection %p",
+        c->channel.bev.get());
   } else {
-    this->log(INFO, "Client on fd %d disconnected", fd);
+    this->log(INFO, "Disconnecting client on fd %d",
+        bufferevent_getfd(c->channel.bev.get()));
   }
 
-  // if the output buffer is not empty, move the client into the draining pool
-  // instead of disconnecting it, to make sure all the data gets sent
-  struct evbuffer* out_buffer = bufferevent_get_output(bev);
-  if (evbuffer_get_length(out_buffer) == 0) {
-    bufferevent_flush(bev, EV_WRITE, BEV_FINISHED);
-    bufferevent_free(bev);
-  } else {
-    // the callbacks will free it when all the data is sent or the client
-    // disconnects
-    bufferevent_setcb(bev, nullptr,
-        Server::dispatch_on_disconnecting_client_output,
-        Server::dispatch_on_disconnecting_client_error, this);
-    bufferevent_disable(bev, EV_READ);
-  }
+  this->channel_to_client.erase(&c->channel);
+  c->channel.disconnect();
 
-  process_disconnect(this->state, c);
+  try {
+    process_disconnect(this->state, c);
+  } catch (const exception& e) {
+    this->log(WARNING, "Error during client disconnect cleanup: %s", e.what());
+  }
+  // c is destroyed here (process_disconnect should remove any other references
+  // to it, e.g. from Lobby objects)
 }
 
 void Server::dispatch_on_listen_accept(
@@ -73,25 +61,6 @@ void Server::dispatch_on_listen_error(struct evconnlistener* listener,
   reinterpret_cast<Server*>(ctx)->on_listen_error(listener);
 }
 
-void Server::dispatch_on_client_input(struct bufferevent* bev, void* ctx) {
-  reinterpret_cast<Server*>(ctx)->on_client_input(bev);
-}
-
-void Server::dispatch_on_client_error(struct bufferevent* bev, short events,
-    void* ctx) {
-  reinterpret_cast<Server*>(ctx)->on_client_error(bev, events);
-}
-
-void Server::dispatch_on_disconnecting_client_output(struct bufferevent* bev,
-    void* ctx) {
-  reinterpret_cast<Server*>(ctx)->on_disconnecting_client_output(bev);
-}
-
-void Server::dispatch_on_disconnecting_client_error(struct bufferevent* bev,
-    short events, void* ctx) {
-  reinterpret_cast<Server*>(ctx)->on_disconnecting_client_error(bev, events);
-}
-
 void Server::on_listen_accept(struct evconnlistener* listener,
     evutil_socket_t fd, struct sockaddr*, int) {
 
@@ -111,15 +80,19 @@ void Server::on_listen_accept(struct evconnlistener* listener,
 
   struct bufferevent *bev = bufferevent_socket_new(this->base.get(), fd,
       BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS);
-  shared_ptr<Client> c(new Client(bev, listening_socket->version,
-      listening_socket->behavior));
-  this->bev_to_client.emplace(make_pair(bev, c));
+  shared_ptr<Client> c(new Client(
+      bev, listening_socket->version, listening_socket->behavior));
+  c->channel.on_command_received = Server::on_client_input;
+  c->channel.on_error = Server::on_client_error;
+  c->channel.context_obj = this;
+  this->channel_to_client.emplace(&c->channel, c);
 
-  bufferevent_setcb(bev, &Server::dispatch_on_client_input, nullptr,
-      &Server::dispatch_on_client_error, this);
-  bufferevent_enable(bev, EV_READ | EV_WRITE);
-
-  process_connect(this->state, c);
+  try {
+    process_connect(this->state, c);
+  } catch (const exception& e) {
+    this->log(WARNING, "Error during client initialization: %s", e.what());
+    this->disconnect_client(c);
+  }
 }
 
 void Server::connect_client(
@@ -128,20 +101,25 @@ void Server::connect_client(
   this->log(INFO, "Client connected on virtual connection %p", bev);
 
   shared_ptr<Client> c(new Client(bev, version, initial_state));
-  this->bev_to_client.emplace(make_pair(bev, c));
+  c->channel.on_command_received = Server::on_client_input;
+  c->channel.on_error = Server::on_client_error;
+  c->channel.context_obj = this;
+
+  this->channel_to_client.emplace(&c->channel, c);
 
   // Manually set the remote address, since the bufferevent has no fd and the
-  // Client constructor can't figure out the virtual remote address
-  auto* sin = reinterpret_cast<sockaddr_in*>(&c->remote_addr);
+  // Channel constructor can't figure out the virtual remote address
+  auto* sin = reinterpret_cast<sockaddr_in*>(&c->channel.remote_addr);
   sin->sin_family = AF_INET;
   sin->sin_addr.s_addr = htonl(address);
   sin->sin_port = htons(port);
 
-  bufferevent_setcb(bev, &Server::dispatch_on_client_input, nullptr,
-      &Server::dispatch_on_client_error, this);
-  bufferevent_enable(bev, EV_READ | EV_WRITE);
-
-  process_connect(this->state, c);
+  try {
+    process_connect(this->state, c);
+  } catch (const exception& e) {
+    this->log(WARNING, "Error during client initialization: %s", e.what());
+    this->disconnect_client(c);
+  }
 }
 
 void Server::on_listen_error(struct evconnlistener* listener) {
@@ -151,73 +129,36 @@ void Server::on_listen_error(struct evconnlistener* listener) {
   event_base_loopexit(this->base.get(), nullptr);
 }
 
-void Server::on_client_input(struct bufferevent* bev) {
-  shared_ptr<Client> c;
-  try {
-    c = this->bev_to_client.at(bev);
-  } catch (const out_of_range& e) {
-    this->log(WARNING, "Received message from client with no configuration");
-
-    // ignore all the data
-    // TODO: we probably should disconnect them or something
-    struct evbuffer* in_buffer = bufferevent_get_input(bev);
-    evbuffer_drain(in_buffer, evbuffer_get_length(in_buffer));
-    return;
-  }
+void Server::on_client_input(Channel& ch, uint16_t command, uint32_t flag, std::string& data) {
+  Server* server = reinterpret_cast<Server*>(ch.context_obj);
+  shared_ptr<Client> c = server->channel_to_client.at(&ch);
 
   if (c->should_disconnect) {
-    this->disconnect_client(bev);
-    return;
-  }
-
-  c->last_recv_time = now();
-  this->receive_and_process_commands(c);
-
-  if (c->should_disconnect) {
-    this->disconnect_client(bev);
-    return;
+    server->disconnect_client(c);
+  } else {
+    try {
+      process_command(server->state, c, command, flag, data);
+    } catch (const exception& e) {
+      server->log(WARNING, "Error processing client command: %s", e.what());
+      c->should_disconnect = true;
+    }
+    if (c->should_disconnect) {
+      server->disconnect_client(c);
+    }
   }
 }
 
-void Server::on_disconnecting_client_output(struct bufferevent* bev) {
-  bufferevent_flush(bev, EV_WRITE, BEV_FINISHED);
-  bufferevent_free(bev);
-}
+void Server::on_client_error(Channel& ch, short events) {
+  Server* server = reinterpret_cast<Server*>(ch.context_obj);
+  shared_ptr<Client> c = server->channel_to_client.at(&ch);
 
-void Server::on_client_error(struct bufferevent* bev, short events) {
   if (events & BEV_EVENT_ERROR) {
     int err = EVUTIL_SOCKET_ERROR();
-    this->log(WARNING, "Client caused error %d (%s)", err,
+    server->log(WARNING, "Client caused error %d (%s)", err,
         evutil_socket_error_to_string(err));
   }
   if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
-    this->disconnect_client(bev);
-  }
-}
-
-void Server::on_disconnecting_client_error(struct bufferevent* bev,
-    short events) {
-  if (events & BEV_EVENT_ERROR) {
-    int err = EVUTIL_SOCKET_ERROR();
-    this->log(WARNING, "Disconnecting client caused error %d (%s)", err,
-        evutil_socket_error_to_string(err));
-  }
-  if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
-    bufferevent_flush(bev, EV_WRITE, BEV_FINISHED);
-    bufferevent_free(bev);
-  }
-}
-
-void Server::receive_and_process_commands(shared_ptr<Client> c) {
-  try {
-    for_each_received_command(c->bev, c->version, c->crypt_in.get(),
-        [this, c](uint16_t command, uint16_t flag, const std::string& data) {
-          process_command(this->state, c, command, flag, data);
-        });
-  } catch (const exception& e) {
-    this->log(INFO, "Error in client stream: %s", e.what());
-    c->should_disconnect = true;
-    return;
+    server->disconnect_client(c);
   }
 }
 
@@ -273,3 +214,13 @@ void Server::add_socket(
   this->listening_sockets.emplace(piecewise_construct, forward_as_tuple(fd),
       forward_as_tuple(this, name, fd, version, behavior));
 }
+
+shared_ptr<Client> Server::get_client() const {
+  if (this->channel_to_client.empty()) {
+    throw runtime_error("no clients on game server");
+  }
+  if (this->channel_to_client.size() > 1) {
+    throw runtime_error("multiple clients on game server");
+  }
+  return this->channel_to_client.begin()->second;
+}

src/Server.hh

@@ -29,6 +29,8 @@ public:
   void connect_client(struct bufferevent* bev, uint32_t address, uint16_t port,
       GameVersion version, ServerBehavior initial_state);
 
+  std::shared_ptr<Client> get_client() const;
+
 private:
   PrefixedLogger log;
   std::shared_ptr<struct event_base> base;
@@ -48,31 +50,20 @@ private:
         ServerBehavior behavior);
   };
   std::unordered_map<int, ListeningSocket> listening_sockets;
-  std::unordered_map<struct bufferevent*, std::shared_ptr<Client>> bev_to_client;
+  std::unordered_map<Channel*, std::shared_ptr<Client>> channel_to_client;
 
   std::shared_ptr<ServerState> state;
 
   static void dispatch_on_listen_accept(struct evconnlistener* listener,
       evutil_socket_t fd, struct sockaddr *address, int socklen, void* ctx);
   static void dispatch_on_listen_error(struct evconnlistener* listener, void* ctx);
-  static void dispatch_on_client_input(struct bufferevent* bev, void* ctx);
-  static void dispatch_on_client_error(struct bufferevent* bev, short events,
-      void* ctx);
-  static void dispatch_on_disconnecting_client_output(struct bufferevent* bev,
-      void* ctx);
-  static void dispatch_on_disconnecting_client_error(struct bufferevent* bev,
-      short events, void* ctx);
 
-  void disconnect_client(struct bufferevent* bev);
   void disconnect_client(std::shared_ptr<Client> c);
 
   void on_listen_accept(struct evconnlistener* listener, evutil_socket_t fd,
       struct sockaddr *address, int socklen);
   void on_listen_error(struct evconnlistener* listener);
-  void on_client_input(struct bufferevent* bev);
-  void on_client_error(struct bufferevent* bev, short events);
-  void on_disconnecting_client_output(struct bufferevent* bev);
-  void on_disconnecting_client_error(struct bufferevent* bev, short events);
 
-  void receive_and_process_commands(std::shared_ptr<Client> c);
+  static void on_client_input(Channel& ch, uint16_t command, uint32_t flag, std::string& data);
+  static void on_client_error(Channel& ch, short events);
 };

src/ServerShell.cc

@@ -125,11 +125,10 @@ Proxy commands (these will only work when exactly one client is connected):\n\
     Enable or disable chat filtering (enabled by default). Chat filtering\n\
     applies newserv\'s standard character replacements to chat messages; for\n\
     example, $ becomes a tab character and # becomes a newline.\n\
-  set-chat-safety <on|off>\n\
-    Enable or disable chat safety (enabled by default). When chat safety is on,\n\
-    all chat messages that begin with a $ are not sent to the remote server.\n\
-    This can prevent embarrassing situations if the remote server isn\'t a\n\
-    newserv instance and you have newserv commands in your chat shortcuts.\n\
+  set-infinite-hp <on|off>\n\
+  set-infinite-tp <on|off>\n\
+    Enable or disable infinite HP or TP. When infinite HP is enabled, attacks\n\
+    that would kill you in one hit will still do so.\n\
   set-switch-assist <on|off>\n\
     Enable or disable switch assist. When switch assist is on, the proxy will\n\
     remember the last \"enable switch\" command that you send, and will send it\n\
@@ -146,6 +145,8 @@ Proxy commands (these will only work when exactly one client is connected):\n\
     responds as if the function was called (with the given return value), but\n\
     does not send the code to the client. To stop blocking function calls, omit\n\
     the return value.\n\
+  close-idle-sessions\n\
+    Closes all sessions that don\'t have a client and server connected.\n\
 ");
 
 
@@ -278,9 +279,6 @@ Proxy commands (these will only work when exactly one client is connected):\n\
   // PROXY COMMANDS
 
   } else if ((command_name == "sc") || (command_name == "ss")) {
-    auto session = this->get_proxy_session();
-
-    bool to_server = (command_name[1] == 's');
     string data = parse_data_string(command_args);
     if (data.size() & 3) {
       throw invalid_argument("data size is not a multiple of 4");
@@ -289,7 +287,26 @@ Proxy commands (these will only work when exactly one client is connected):\n\
       throw invalid_argument("no data given");
     }
 
-    session->send_to_end_with_header(to_server, data);
+    shared_ptr<ProxyServer::LinkedSession> proxy_session;
+    try {
+      proxy_session = this->get_proxy_session();
+    } catch (const exception&) { }
+
+    if (proxy_session.get()) {
+      if (command_name[1] == 's') {
+        proxy_session->server_channel.send(data);
+      } else {
+        proxy_session->client_channel.send(data);
+      }
+
+    } else {
+      if (command_name [1] == 's') {
+        throw runtime_error("cannot send to server in non-proxy session");
+      }
+
+      auto c = this->state->game_server->get_client();
+      send_command_with_header(c, data.data(), data.size());
+    }
 
   } else if ((command_name == "chat") || (command_name == "dchat")) {
     auto session = this->get_proxy_session();
@@ -305,11 +322,11 @@ Proxy commands (these will only work when exactly one client is connected):\n\
     data.push_back('\0');
     data.resize((data.size() + 3) & (~3));
 
-    session->send_to_end(true, 0x06, 0x00, data);
+    session->server_channel.send(0x06, 0x00, data);
 
   } else if (command_name == "marker") {
     auto session = this->get_proxy_session();
-    session->send_to_end(true, 0x89, stoul(command_args));
+    session->server_channel.send(0x89, stoul(command_args));
 
   } else if (command_name == "warp") {
     auto session = this->get_proxy_session();
@@ -319,8 +336,8 @@ Proxy commands (these will only work when exactly one client is connected):\n\
     cmds[0].word[1] = session->lobby_client_id;
     cmds[1].dword = stoul(command_args);
 
-    session->send_to_end(false, 0x60, 0x00, &cmds, sizeof(cmds));
-    session->send_to_end(true, 0x60, 0x00, &cmds, sizeof(cmds));
+    session->client_channel.send(0x60, 0x00, &cmds, sizeof(cmds));
+    session->server_channel.send(0x60, 0x00, &cmds, sizeof(cmds));
 
   } else if ((command_name == "info-board") || (command_name == "info-board-data")) {
     auto session = this->get_proxy_session();
@@ -334,7 +351,7 @@ Proxy commands (these will only work when exactly one client is connected):\n\
     data.push_back('\0');
     data.resize((data.size() + 3) & (~3));
 
-    session->send_to_end(true, 0xD9, 0x00, data);
+    session->server_channel.send(0xD9, 0x00, data);
 
   } else if (command_name == "set-override-section-id") {
     auto session = this->get_proxy_session();
@@ -350,7 +367,7 @@ Proxy commands (these will only work when exactly one client is connected):\n\
       session->override_lobby_event = -1;
     } else {
       session->override_lobby_event = event_for_name(command_args);
-      session->send_to_end(false, 0xDA, session->override_lobby_event);
+      session->client_channel.send(0xDA, session->override_lobby_event);
     }
 
   } else if (command_name == "set-override-lobby-number") {
@@ -365,13 +382,17 @@ Proxy commands (these will only work when exactly one client is connected):\n\
     auto session = this->get_proxy_session();
     set_boolean(&session->enable_chat_filter, command_args);
 
-  } else if (command_name == "set-chat-safety") {
+  } else if (command_name == "set-infinite-hp") {
     auto session = this->get_proxy_session();
-    set_boolean(&session->suppress_newserv_commands, command_args);
+    set_boolean(&session->infinite_hp, command_args);
+
+  } else if (command_name == "set-infinite-tp") {
+    auto session = this->get_proxy_session();
+    set_boolean(&session->infinite_tp, command_args);
 
   } else if (command_name == "set-switch-assist") {
     auto session = this->get_proxy_session();
-    set_boolean(&session->enable_switch_assist, command_args);
+    set_boolean(&session->switch_assist, command_args);
 
   } else if (command_name == "set-save-files") {
     auto session = this->get_proxy_session();
@@ -385,6 +406,10 @@ Proxy commands (these will only work when exactly one client is connected):\n\
       session->function_call_return_value = stoul(command_args);
     }
 
+  } else if (command_name == "close-idle-sessions") {
+    size_t count = this->state->proxy_server->delete_disconnected_sessions();
+    fprintf(stderr, "%zu sessions closed\n", count);
+
   } else {
     throw invalid_argument("unknown command; try \'help\'");
   }

src/ServerState.cc

@@ -3,10 +3,11 @@
 #include <string.h>
 
 #include <memory>
+#include <phosg/Network.hh>
 
-#include "SendCommands.hh"
-#include "NetworkAddresses.hh"
 #include "IPStackSimulator.hh"
+#include "NetworkAddresses.hh"
+#include "SendCommands.hh"
 #include "Text.hh"
 
 using namespace std;
@@ -17,6 +18,7 @@ ServerState::ServerState()
   : dns_server_port(0),
     ip_stack_debug(false),
     allow_unregistered_users(false),
+    item_tracking_enabled(true),
     run_shell_behavior(RunShellBehavior::DEFAULT), next_lobby_id(1),
     pre_lobby_event(0),
     ep3_menu_song(-1) {
@@ -174,17 +176,17 @@ shared_ptr<Client> ServerState::find_client(const std::u16string* identifier,
 }
 
 uint32_t ServerState::connect_address_for_client(std::shared_ptr<Client> c) {
-  if (c->is_virtual_connection) {
-    if (c->remote_addr.ss_family != AF_INET) {
+  if (c->channel.is_virtual_connection) {
+    if (c->channel.remote_addr.ss_family != AF_INET) {
       throw logic_error("virtual connection is missing remote IPv4 address");
     }
-    const auto* sin = reinterpret_cast<const sockaddr_in*>(&c->remote_addr);
+    const auto* sin = reinterpret_cast<const sockaddr_in*>(&c->channel.remote_addr);
     return IPStackSimulator::connect_address_for_remote_address(
         ntohl(sin->sin_addr.s_addr));
   } else {
     // TODO: we can do something smarter here, like use the sockname to find
     // out which interface the client is connected to, and return that address
-    if (is_local_address(c->remote_addr)) {
+    if (is_local_address(c->channel.remote_addr)) {
       return this->local_address;
     } else {
       return this->external_address;
@@ -237,3 +239,119 @@ void ServerState::set_port_configuration(
     }
   }
 }
+
+
+
+void ServerState::create_menus(shared_ptr<const JSONObject> config_json) {
+  const auto& d = config_json->as_dict();
+
+  shared_ptr<vector<MenuItem>> information_menu_pc(new vector<MenuItem>());
+  shared_ptr<vector<MenuItem>> information_menu_gc(new vector<MenuItem>());
+  shared_ptr<vector<u16string>> information_contents(new vector<u16string>());
+
+  information_menu_gc->emplace_back(InformationMenuItemID::GO_BACK, u"Go back",
+      u"Return to the\nmain menu", 0);
+  {
+    uint32_t item_id = 0;
+    for (const auto& item : d.at("InformationMenuContents")->as_list()) {
+      auto& v = item->as_list();
+      information_menu_pc->emplace_back(item_id, decode_sjis(v.at(0)->as_string()),
+          decode_sjis(v.at(1)->as_string()), 0);
+      information_menu_gc->emplace_back(item_id, decode_sjis(v.at(0)->as_string()),
+          decode_sjis(v.at(1)->as_string()), MenuItem::Flag::REQUIRES_MESSAGE_BOXES);
+      information_contents->emplace_back(decode_sjis(v.at(2)->as_string()));
+      item_id++;
+    }
+  }
+  this->information_menu_pc = information_menu_pc;
+  this->information_menu_gc = information_menu_gc;
+  this->information_contents = information_contents;
+
+  auto generate_proxy_destinations_menu = +[](
+      vector<MenuItem>& ret_menu,
+      vector<pair<string, uint16_t>>& ret_pds,
+      const unordered_map<string, shared_ptr<JSONObject>>& d) {
+    ret_menu.clear();
+    ret_pds.clear();
+
+    ret_menu.emplace_back(ProxyDestinationsMenuItemID::GO_BACK, u"Go back",
+        u"Return to the\nmain menu", 0);
+
+    uint32_t item_id = 0;
+    for (const auto& item : d) {
+      const string& netloc_str = item.second->as_string();
+      const string& description = "$C7Remote server:\n$C6" + netloc_str;
+      ret_menu.emplace_back(item_id, decode_sjis(item.first),
+          decode_sjis(description), 0);
+      ret_pds.emplace_back(parse_netloc(netloc_str));
+      item_id++;
+    }
+  };
+
+  generate_proxy_destinations_menu(
+      this->proxy_destinations_menu_pc,
+      this->proxy_destinations_pc,
+      d.at("ProxyDestinations-PC")->as_dict());
+  generate_proxy_destinations_menu(
+      this->proxy_destinations_menu_gc,
+      this->proxy_destinations_gc,
+      d.at("ProxyDestinations-GC")->as_dict());
+
+  try {
+    const string& netloc_str = d.at("ProxyDestination-Patch")->as_string();
+    this->proxy_destination_patch = parse_netloc(netloc_str);
+    log(INFO, "Patch server proxy is enabled with destination %s", netloc_str.c_str());
+    for (auto& it : this->name_to_port_config) {
+      if (it.second->version == GameVersion::PATCH) {
+        it.second->behavior = ServerBehavior::PROXY_SERVER;
+      }
+    }
+  } catch (const out_of_range&) {
+    this->proxy_destination_patch.first = "";
+    this->proxy_destination_patch.second = 0;
+  }
+  try {
+    const string& netloc_str = d.at("ProxyDestination-BB")->as_string();
+    this->proxy_destination_bb = parse_netloc(netloc_str);
+    log(INFO, "BB proxy is enabled with destination %s", netloc_str.c_str());
+    for (auto& it : this->name_to_port_config) {
+      if (it.second->version == GameVersion::BB) {
+        it.second->behavior = ServerBehavior::PROXY_SERVER;
+      }
+    }
+  } catch (const out_of_range&) {
+    this->proxy_destination_bb.first = "";
+    this->proxy_destination_bb.second = 0;
+  }
+
+  this->main_menu.emplace_back(MainMenuItemID::GO_TO_LOBBY, u"Go to lobby",
+      u"Join the lobby", 0);
+  this->main_menu.emplace_back(MainMenuItemID::INFORMATION, u"Information",
+      u"View server\ninformation", MenuItem::Flag::REQUIRES_MESSAGE_BOXES);
+  if (!this->proxy_destinations_pc.empty()) {
+    this->main_menu.emplace_back(MainMenuItemID::PROXY_DESTINATIONS, u"Proxy server",
+        u"Connect to another\nserver", MenuItem::Flag::PC_ONLY);
+  }
+  if (!this->proxy_destinations_gc.empty()) {
+    this->main_menu.emplace_back(MainMenuItemID::PROXY_DESTINATIONS, u"Proxy server",
+        u"Connect to another\nserver", MenuItem::Flag::GC_ONLY);
+  }
+  this->main_menu.emplace_back(MainMenuItemID::DOWNLOAD_QUESTS, u"Download quests",
+      u"Download quests", MenuItem::Flag::INVISIBLE_ON_BB);
+  if (!this->dol_file_index->empty()) {
+    this->main_menu.emplace_back(MainMenuItemID::PATCHES, u"Patches",
+        u"Change game\nbehaviors", MenuItem::Flag::GC_ONLY | MenuItem::Flag::REQUIRES_SEND_FUNCTION_CALL);
+  }
+  if (!this->dol_file_index->empty()) {
+    this->main_menu.emplace_back(MainMenuItemID::PROGRAMS, u"Programs",
+        u"Run GameCube\nprograms", MenuItem::Flag::GC_ONLY | MenuItem::Flag::REQUIRES_SEND_FUNCTION_CALL | MenuItem::Flag::REQUIRES_SAVE_DISABLED);
+  }
+  this->main_menu.emplace_back(MainMenuItemID::DISCONNECT, u"Disconnect",
+      u"Disconnect", 0);
+  this->main_menu.emplace_back(MainMenuItemID::CLEAR_LICENSE, u"Clear license",
+      u"Disconnect with an\ninvalid license error\nso you can enter a\ndifferent serial\nnumber, access key,\nor password", 0);
+
+  try {
+    this->welcome_message = decode_sjis(d.at("WelcomeMessage")->as_string());
+  } catch (const out_of_range&) { }
+}

src/ServerState.hh

@@ -2,13 +2,15 @@
 
 #include <atomic>
 #include <map>
-#include <unordered_map>
-#include <string>
 #include <memory>
-#include <vector>
+#include <phosg/JSON.hh>
 #include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "Client.hh"
+#include "FunctionCompiler.hh"
 #include "Items.hh"
 #include "LevelTable.hh"
 #include "License.hh"
@@ -18,8 +20,9 @@
 
 
 
-// Forwawrd declaration due to reference cycle
+// Forwawrd declarations due to reference cycles
 class ProxyServer;
+class Server;
 
 struct PortConfiguration {
   std::string name;
@@ -43,8 +46,12 @@ struct ServerState {
   std::vector<std::string> ip_stack_addresses;
   bool ip_stack_debug;
   bool allow_unregistered_users;
+  bool item_tracking_enabled;
   RunShellBehavior run_shell_behavior;
   std::vector<std::shared_ptr<const PSOBBEncryption::KeyFile>> bb_private_keys;
+  std::shared_ptr<const FunctionCodeIndex> function_code_index;
+  std::shared_ptr<const DOLFileIndex> dol_file_index;
+  std::shared_ptr<const Ep3DataIndex> ep3_data_index;
   std::shared_ptr<const QuestIndex> quest_index;
   std::shared_ptr<const LevelTable> level_table;
   std::shared_ptr<const BattleParamTable> battle_params;
@@ -75,9 +82,8 @@ struct ServerState {
   uint32_t local_address;
   uint32_t external_address;
 
-  // TODO: This is only here because the menu selection handler has to call
-  // delete_session on it. Find a cleaner way to do this.
   std::shared_ptr<ProxyServer> proxy_server;
+  std::shared_ptr<Server> game_server;
 
   ServerState();
 
@@ -108,4 +114,6 @@ struct ServerState {
 
   void set_port_configuration(
     const std::vector<PortConfiguration>& port_configs);
+
+  void create_menus(std::shared_ptr<const JSONObject> config_json);
 };

src/StaticGameData.cc

@@ -1,5 +1,7 @@
 #include "StaticGameData.hh"
 
+#include <array>
+
 using namespace std;
 
 
@@ -141,8 +143,9 @@ u16string u16name_for_section_id(uint8_t section_id) {
 }
 
 uint8_t section_id_for_name(const string& name) {
+  string lower_name = tolower(name);
   try {
-    return name_to_section_id.at(name);
+    return name_to_section_id.at(lower_name);
   } catch (const out_of_range&) { }
   try {
     uint64_t x = stoul(name);
@@ -253,24 +256,112 @@ uint8_t npc_for_name(const u16string& name) {
 
 
 
-const unordered_map<uint32_t, uint32_t> combine_item_to_max({
-  {0x030000, 10},
-  {0x030001, 10},
-  {0x030002, 10},
-  {0x030100, 10},
-  {0x030101, 10},
-  {0x030102, 10},
-  {0x030300, 10},
-  {0x030400, 10},
-  {0x030500, 10},
-  {0x030600, 10},
-  {0x030601, 10},
-  {0x030700, 10},
-  {0x030800, 10},
-  {0x031000, 99},
-  {0x031001, 99},
-  {0x031002, 99},
-});
+const char* name_for_char_class(uint8_t cls) {
+  static const array<const char*, 12> names = {
+    "HUmar",
+    "HUnewearl",
+    "HUcast",
+    "RAmar",
+    "RAcast",
+    "RAcaseal",
+    "FOmarl",
+    "FOnewm",
+    "FOnewearl",
+    "HUcaseal",
+    "FOmar",
+    "RAmarl",
+  };
+  try {
+    return names.at(cls);
+  } catch (const out_of_range&) {
+    return "Unknown";
+  }
+}
+
+const char* abbreviation_for_char_class(uint8_t cls) {
+  static const array<const char*, 12> names = {
+    "HUmr",
+    "HUnl",
+    "HUcs",
+    "RAmr",
+    "RAcs",
+    "RAcl",
+    "FOml",
+    "FOnm",
+    "FOnl",
+    "HUcl",
+    "FOmr",
+    "RAml",
+  };
+  try {
+    return names.at(cls);
+  } catch (const out_of_range&) {
+    return "???";
+  }
+}
+
+
+
+const char* name_for_difficulty(uint8_t difficulty) {
+  static const array<const char*, 4> names = {
+    "Normal",
+    "Hard",
+    "Very Hard",
+    "Ultimate",
+  };
+  try {
+    return names.at(difficulty);
+  } catch (const out_of_range&) {
+    return "Unknown";
+  }
+}
+
+char abbreviation_for_difficulty(uint8_t difficulty) {
+  static const array<char, 4> names = {'N', 'H', 'V', 'U'};
+  try {
+    return names.at(difficulty);
+  } catch (const out_of_range&) {
+    return '?';
+  }
+}
+
+
+
+const char* abbreviation_for_game_mode(uint8_t mode) {
+  static const array<const char*, 4> names = {
+    "Nml",
+    "Btl",
+    "Chl",
+    "Solo",
+  };
+  try {
+    return names.at(mode);
+  } catch (const out_of_range&) {
+    return "???";
+  }
+}
+
+
+
+size_t stack_size_for_item(uint8_t data0, uint8_t data1) {
+  if (data0 == 4) {
+    return 999999;
+  }
+  if (data0 == 3) {
+    if ((data1 < 9) && (data1 != 2)) {
+      return 10;
+    } else if (data1 == 0x10) {
+      return 99;
+    }
+  }
+  return 1;
+}
+
+size_t stack_size_for_item(const ItemData& item) {
+  return stack_size_for_item(item.data1[0], item.data1[1]);
+}
+
+
 
 const unordered_map<uint8_t, const char*> name_for_weapon_special({
   {0x00, nullptr},
@@ -1378,6 +1469,8 @@ string name_for_item(const ItemData& item, bool include_color_codes) {
 
   // For weapons, specials appear before the weapon name
   if ((item.data1[0] == 0x00) && (item.data1[4] != 0x00)) {
+    // 0x80 is the unidentified flag, but we always return the identified name
+    // of the item here, so we ignore it
     bool is_present = item.data1[4] & 0x40;
     uint8_t special_id = item.data1[4] & 0x3F;
     if (is_present) {
@@ -1592,7 +1685,7 @@ string name_for_item(const ItemData& item, bool include_color_codes) {
 
   // For tools, add the amount (if applicable)
   } else if (item.data1[0] == 0x03) {
-    if (combine_item_to_max.count(primary_identifier)) {
+    if (stack_size_for_item(item) > 1) {
       ret_tokens.emplace_back(string_printf("x%hhu", item.data1[5]));
     }
   }

src/StaticGameData.hh

@@ -8,7 +8,9 @@
 
 
 
-extern const std::unordered_map<uint32_t, uint32_t> combine_item_to_max;
+size_t stack_size_for_item(uint8_t data0, uint8_t data1);
+size_t stack_size_for_item(const ItemData& item);
+
 extern const std::unordered_map<uint8_t, const char*> name_for_weapon_special;
 extern const std::unordered_map<uint8_t, const char*> name_for_s_rank_special;
 extern const std::unordered_map<uint32_t, const char*> name_for_primary_identifier;
@@ -38,4 +40,12 @@ std::u16string u16name_for_npc(uint8_t npc);
 uint8_t npc_for_name(const std::string& name);
 uint8_t npc_for_name(const std::u16string& name);
 
+const char* name_for_char_class(uint8_t cls);
+const char* abbreviation_for_char_class(uint8_t cls);
+
+const char* name_for_difficulty(uint8_t difficulty);
+char abbreviation_for_difficulty(uint8_t difficulty);
+
+const char* abbreviation_for_game_mode(uint8_t);
+
 std::string name_for_item(const ItemData& item, bool include_color_codes);

src/Text.hh

@@ -5,6 +5,7 @@
 #include <string.h>
 
 #include <string>
+#include <stdexcept>
 #include <phosg/Encoding.hh>
 #include <phosg/Strings.hh>
 
@@ -217,6 +218,19 @@ struct parray {
     return this->items;
   }
 
+  ItemT& operator[](size_t index) {
+    if (index >= Count) {
+      throw std::out_of_range("array index out of bounds");
+    }
+    return this->items[index];
+  }
+  const ItemT& operator[](size_t index) const {
+    if (index >= Count) {
+      throw std::out_of_range("array index out of bounds");
+    }
+    return this->items[index];
+  }
+
   // TODO: These can be made faster by only clearing the unused space after the
   // strncpy_t (if any) instead of clearing all the space every time
   parray& operator=(const parray& s) {
@@ -246,6 +260,9 @@ struct parray {
   }
 
   parray& operator=(const ItemT* s) {
+    if (!s) {
+      throw std::logic_error("attempted to assign nullptr to parray");
+    }
     for (size_t x = 0; x < Count; x++) {
       this->items[x] = s[x];
     }
@@ -274,6 +291,15 @@ struct parray {
       this->items[x] = v;
     }
   }
+
+  bool is_filled_with(ItemT v) const {
+    for (size_t x = 0; x < Count; x++) {
+      if (this->items[x] != v) {
+        return false;
+      }
+    }
+    return true;
+  }
 } __attribute__((packed));
 
 
@@ -293,10 +319,16 @@ struct ptext : parray<CharT, Count> {
 
   template <typename OtherCharT>
   ptext(const OtherCharT* s) {
+    if (!s) {
+      throw std::logic_error("attempted to assign nullptr to ptext");
+    }
     this->operator=(s);
   }
   template <typename OtherCharT>
   ptext(const OtherCharT* s, size_t count) {
+    if (!s) {
+      throw std::logic_error("attempted to assign nullptr to ptext");
+    }
     this->assign(s, count);
   }
   template <typename OtherCharT>
@@ -323,12 +355,18 @@ struct ptext : parray<CharT, Count> {
 
   template <typename OtherCharT>
   ptext& operator=(const OtherCharT* s) {
+    if (!s) {
+      throw std::logic_error("attempted to assign nullptr to ptext");
+    }
     size_t chars_written = text_strncpy_t(this->items, Count, s, Count);
     this->clear_after(chars_written);
     return *this;
   }
   template <typename OtherCharT>
   ptext& assign(const OtherCharT* s, size_t s_count) {
+    if (!s) {
+      throw std::logic_error("attempted to assign nullptr to ptext");
+    }
     size_t chars_written = text_strncpy_t(this->items, Count, s, s_count);
     this->clear_after(chars_written);
     return *this;
@@ -348,6 +386,9 @@ struct ptext : parray<CharT, Count> {
 
   template <typename OtherCharT>
   bool operator==(const OtherCharT* s) const {
+    if (!s) {
+      throw std::logic_error("attempted to compare ptext to nullptr");
+    }
     return text_strneq_t(this->items, s, Count);
   }
   template <typename OtherCharT>
@@ -360,6 +401,9 @@ struct ptext : parray<CharT, Count> {
   }
   template <typename OtherCharT>
   bool operator!=(const OtherCharT* s) const {
+    if (!s) {
+      throw std::logic_error("attempted to compare ptext to nullptr");
+    }
     return !this->operator==(s);
   }
   template <typename OtherCharT>
@@ -373,6 +417,9 @@ struct ptext : parray<CharT, Count> {
 
   template <typename OtherCharT>
   bool eq_n(const OtherCharT* s, size_t count) const {
+    if (!s) {
+      throw std::logic_error("attempted to compare ptext to nullptr");
+    }
     return text_strneq_t(this->items, s, count);
   }
   template <typename OtherCharT>
@@ -385,9 +432,7 @@ struct ptext : parray<CharT, Count> {
   }
 
   operator std::basic_string<CharT>() const {
-    std::basic_string<CharT> ret(this->items, Count);
-    strip_trailing_zeroes(ret);
-    return ret;
+    return std::basic_string<CharT>(this->items, this->len());
   }
 
   bool empty() const {