fix readme

This adds archive links to updated Teth clients to make set up work with newserv more seamlessly. The Teth clients linked:

- Have map files for Coren
- Have fixed unitxt_e.prs and unitxt_j.prs files so no "Revival Curiass" error
- Connect to 127.0.0.1 by default for the common use case of local server on same machine
- Japanese client has had the version string changed to "JTethVer12513" so it doesn't get overriden to English

I will probably update these archives in the future to use the English files properly with the English client but for now this will make things just work in the mean time hopefully.

.github/workflows/cmake.yml

@@ -27,14 +27,23 @@ jobs:
 
       - name: Install libraries (macOS)
         if: ${{ matrix.os == 'macos-latest' }}
-        run: brew install libevent
+        run: |
+          brew install libevent
+
+          cat << EOF > nproc
+          #!/bin/sh
+          sysctl -n hw.logicalcpu
+          EOF
+          chmod a+x nproc
+          sudo cp nproc /usr/local/bin/nproc
+          rm -f nproc
 
       - name: Install phosg
         run: |
           git clone https://github.com/fuzziqersoftware/phosg.git
           cd phosg
           cmake .
-          make
+          make -j $(nproc)
           sudo make install
 
       - name: Install resource_file
@@ -43,14 +52,14 @@ jobs:
           git clone https://github.com/fuzziqersoftware/resource_dasm.git
           cd resource_dasm
           cmake .
-          make
+          make -j $(nproc)
           sudo make install
 
       - name: Configure CMake
         run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
 
       - name: Build
-        run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+        run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j $(nproc)
 
       - name: Test
         working-directory: ${{github.workspace}}/build

.github/workflows/docker.yaml

@@ -0,0 +1,55 @@
+name: Docker
+
+on:
+  # After build passes with tests
+  workflow_run:
+    workflows: [CMake]
+    types: [completed]
+    branches:
+      - master
+
+  push:
+    tags:
+      - 'v**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Build
+    permissions:
+      contents: read
+      id-token: write
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=sha
+            type=ref,event=tag
+            type=semver,pattern={{version}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

CMakeLists.txt

@@ -1,4 +1,5 @@
 cmake_minimum_required(VERSION 3.10)
+set(CMAKE_POLICY_DEFAULT_CMP0110 NEW)
 
 
 
@@ -6,7 +7,7 @@ cmake_minimum_required(VERSION 3.10)
 
 project(newserv)
 
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 if (MSVC)
     add_compile_options(/W4 /WX)
@@ -14,12 +15,6 @@ else()
     add_compile_options(-Wall -Wextra -Werror -Wno-address-of-packed-member)
 endif()
 
-set(LOCAL_INCLUDE_DIR "/usr/local/include")
-set(LOCAL_LIB_DIR "/usr/local/lib")
-list(APPEND CMAKE_PREFIX_PATH ${LOCAL_LIB_DIR})
-include_directories(${LOCAL_INCLUDE_DIR})
-link_directories(${LOCAL_LIB_DIR})
-
 
 
 # Library search
@@ -27,10 +22,12 @@ link_directories(${LOCAL_LIB_DIR})
 find_path     (LIBEVENT_INCLUDE_DIR NAMES event.h)
 find_library  (LIBEVENT_LIBRARY     NAMES event)
 find_library  (LIBEVENT_CORE        NAMES event_core)
+find_library  (LIBEVENT_PTHREADS    NAMES event_pthreads)
 set (LIBEVENT_INCLUDE_DIRS ${LIBEVENT_INCLUDE_DIR})
 set (LIBEVENT_LIBRARIES
         ${LIBEVENT_LIBRARY}
-        ${LIBEVENT_CORE})
+        ${LIBEVENT_CORE}
+        ${LIBEVENT_PTHREADS})
 
 find_package(phosg REQUIRED)
 find_package(Iconv REQUIRED)
@@ -56,6 +53,7 @@ add_custom_target(
 
 set(SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Revision.cc
+    src/Account.cc
     src/AFSArchive.cc
     src/BattleParamsIndex.cc
     src/BMLArchive.cc
@@ -68,6 +66,7 @@ set(SOURCES
     src/Compression.cc
     src/DCSerialNumbers.cc
     src/DNSServer.cc
+    src/DownloadSession.cc
     src/EnemyType.cc
     src/Episode3/AssistServer.cc
     src/Episode3/BattleRecord.cc
@@ -81,19 +80,22 @@ set(SOURCES
     src/Episode3/RulerServer.cc
     src/Episode3/Server.cc
     src/Episode3/Tournament.cc
+    src/EventUtils.cc
     src/FileContentsCache.cc
     src/FunctionCompiler.cc
     src/GSLArchive.cc
     src/GVMEncoder.cc
+    src/HTTPServer.cc
+    src/IntegralExpression.cc
     src/IPFrameInfo.cc
     src/IPStackSimulator.cc
+    src/IPV4RangeSet.cc
     src/ItemCreator.cc
     src/ItemData.cc
     src/ItemNameIndex.cc
     src/ItemParameterTable.cc
     src/Items.cc
     src/LevelTable.cc
-    src/License.cc
     src/Lobby.cc
     src/Loggers.cc
     src/Main.cc
@@ -101,6 +103,7 @@ set(SOURCES
     src/Menu.cc
     src/NetworkAddresses.cc
     src/PatchFileIndex.cc
+    src/PatchServer.cc
     src/PlayerFilesManager.cc
     src/PlayerSubordinates.cc
     src/ProxyCommands.cc
@@ -109,7 +112,6 @@ set(SOURCES
     src/PSOGCObjectGraph.cc
     src/PSOProtocol.cc
     src/Quest.cc
-    src/QuestAvailabilityExpression.cc
     src/QuestScript.cc
     src/RareItemSet.cc
     src/ReceiveCommands.cc
@@ -121,9 +123,8 @@ set(SOURCES
     src/Server.cc
     src/ServerShell.cc
     src/ServerState.cc
-    src/Shell.cc
+    src/SignalWatcher.cc
     src/StaticGameData.cc
-    src/StepGraph.cc
     src/TeamIndex.cc
     src/Text.cc
     src/TextIndex.cc
@@ -132,21 +133,23 @@ set(SOURCES
 )
 
 if(resource_file_FOUND)
-    set(SOURCES ${SOURCES} src/ARCodeTranslator.cc)
+    set(SOURCES ${SOURCES} src/AddressTranslator.cc)
 endif()
 
 add_executable(newserv ${SOURCES})
 target_include_directories(newserv PUBLIC ${LIBEVENT_INCLUDE_DIR} ${Iconv_INCLUDE_DIRS})
-target_link_libraries(newserv phosg ${LIBEVENT_LIBRARIES} ${Iconv_LIBRARIES} pthread)
-add_dependencies(newserv newserv-Revision-cc)
-
+target_link_libraries(newserv phosg::phosg ${LIBEVENT_LIBRARIES} ${Iconv_LIBRARIES} pthread)
 if(resource_file_FOUND)
     target_compile_definitions(newserv PUBLIC HAVE_RESOURCE_FILE)
-    target_link_libraries(newserv resource_file)
-    message(STATUS "libresource_file found; enabling patch support")
+    target_link_libraries(newserv resource_file::resource_file)
+    message(STATUS "resource_file found; enabling patch support")
 else()
-    message(WARNING "libresource_file not found; disabling patch support")
+    message(WARNING "resource_file not found; disabling patch support")
 endif()
+add_dependencies(newserv newserv-Revision-cc)
+
+# target_compile_options(newserv PRIVATE -fsanitize=address)
+# target_link_options(newserv PRIVATE -fsanitize=address)
 
 
 
@@ -155,6 +158,7 @@ endif()
 enable_testing()
 
 file(GLOB LogTestCases ${CMAKE_SOURCE_DIR}/tests/*.test.txt)
+file(GLOB LogRDTestCases ${CMAKE_SOURCE_DIR}/tests/*.rdtest.txt)
 
 foreach(LogTestCase IN ITEMS ${LogTestCases})
     add_test(
@@ -163,6 +167,15 @@ foreach(LogTestCase IN ITEMS ${LogTestCases})
         COMMAND ${CMAKE_BINARY_DIR}/newserv --replay-log=${LogTestCase} --config=${CMAKE_SOURCE_DIR}/tests/config.json)
 endforeach()
 
+if(resource_file_FOUND)
+    foreach(LogRDTestCase IN ITEMS ${LogRDTestCases})
+        add_test(
+            NAME ${LogRDTestCase}
+            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+            COMMAND ${CMAKE_BINARY_DIR}/newserv --replay-log=${LogRDTestCase} --config=${CMAKE_SOURCE_DIR}/tests/config.json)
+    endforeach()
+endif()
+
 file(GLOB ScriptTestCases ${CMAKE_SOURCE_DIR}/tests/*.test.sh)
 
 foreach(ScriptTestCase IN ITEMS ${ScriptTestCases})
@@ -172,6 +185,8 @@ foreach(ScriptTestCase IN ITEMS ${ScriptTestCases})
         COMMAND ${ScriptTestCase} ${CMAKE_BINARY_DIR}/newserv)
 endforeach()
 
+
+
 # Installation configuration
 
 install(TARGETS newserv DESTINATION bin)

Dockerfile

@@ -0,0 +1,87 @@
+# syntax=docker/dockerfile:1
+
+ARG BASE_IMAGE=ubuntu:24.04
+FROM ${BASE_IMAGE} AS builder
+
+RUN apt update && apt install -y --no-install-recommends \
+    python3 \
+    git \
+    ca-certificates \
+    sudo \
+    make \
+    cmake \
+    g++ \
+    libevent-dev \
+    zlib1g-dev
+
+# ---
+
+FROM builder AS deps
+
+ARG PHOSG_TARGET=master
+ARG RESOURCE_DASM_TARGET=master
+ARG BUILD_RESOURCE_DASM=true
+
+RUN git clone --depth 1 -b ${PHOSG_TARGET} https://github.com/fuzziqersoftware/phosg.git && \
+    cd phosg && \
+    cmake . && \
+    make -j$(nproc) && \
+    sudo make install
+
+RUN \
+if [ "$BUILD_RESOURCE_DASM" = "true" ] ; then \
+    git clone --depth 1 -b ${RESOURCE_DASM_TARGET} https://github.com/fuzziqersoftware/resource_dasm.git && \
+    cd resource_dasm && \
+    cmake . && \
+    make -j$(nproc) && \
+    sudo make install \
+; fi
+
+# ---
+
+FROM builder AS newserv
+
+ARG BUILD_TYPE=Release
+ARG BUILD_STRIP=true
+
+WORKDIR /usr/src/newserv
+COPY . .
+COPY --from=deps /usr/local /usr/local
+
+RUN cmake -B $PWD/build -DCMAKE_BUILD_TYPE=${BUILD_TYPE} && \
+    cmake --build $PWD/build --config ${BUILD_TYPE} -j $(nproc) && \
+    sudo make -C build install
+
+RUN \
+if [ "$BUILD_STRIP" = "true" ] ; then \
+  strip /usr/local/lib/*.a && \
+  strip /usr/local/bin/* \
+; fi
+
+# ---
+
+FROM ${BASE_IMAGE} AS data
+
+WORKDIR /newserv
+COPY system/ ./system
+RUN cp -f system/config.example.json system/config.json && \
+    sed -i 's/"ExternalAddress": "[^"]*"/"ExternalAddress": "0.0.0.0"/' system/config.json
+
+# ---
+
+FROM ${BASE_IMAGE} AS final
+
+RUN apt update && apt install -y --no-install-recommends \
+    libevent-dev \
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+WORKDIR /newserv
+COPY --from=data /newserv .
+COPY --from=newserv /usr/local /usr/local
+
+USER root
+VOLUME /newserv/system
+
+# does not allow receiving any signal at the moment, so force kill the app
+STOPSIGNAL SIGKILL
+CMD ["newserv"]

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2023 Martin Michelsen
+Copyright (c) 2024 Martin Michelsen
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -18,4 +18,3 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-

README.md

@@ -1,6 +1,6 @@
-# newserv <img align="right" src="s-newserv.png" />
+# newserv <img align="right" src="static/s-newserv.png" />
 
-newserv is a game server, proxy, and reverse-engineering tool for Phantasy Star Online (PSO).
+newserv is a game server, proxy, and reverse-engineering tool for Phantasy Star Online (PSO). **To quickly get started using newserv, just read the [server setup](#server-setup) and [how to connect](#how-to-connect) sections.**
 
 This project includes code that was reverse-engineered by the community in ages long past, and has been included in many projects since then. It also includes some game data from Phantasy Star Online itself, which was originally created by Sega.
 
@@ -9,46 +9,112 @@ Feel free to submit GitHub issues if you find bugs or have feature requests. I'd
 See TODO.md for a list of known issues and future work I've curated, or go to the GitHub issue tracker for issues and requests submitted by the community.
 
 **Table of contents**
+* Background
+    * [History](#history)
+    * [Other server projects](#other-server-projects)
+    * [Using newserv in other projects](#using-newserv-in-other-projects)
+    * [Developer information](#developer-information)
 * [Compatibility](#compatibility)
 * Setup
     * [Server setup](#server-setup)
     * [Client patch directories for PC and BB](#client-patch-directories)
     * [How to connect](#how-to-connect)
 * Features and configuration
+    * [User accounts](#user-accounts)
     * [Installing quests](#installing-quests)
     * [Item tables and drop modes](#item-tables-and-drop-modes)
     * [Cross-version play](#cross-version-play)
+    * [Server-side saves](#server-side-saves)
     * [Episode 3 features](#episode-3-features)
     * [Memory patches, client functions, and DOL files](#memory-patches-client-functions-and-dol-files)
     * [Using newserv as a proxy](#using-newserv-as-a-proxy)
     * [Chat commands](#chat-commands)
 * [Non-server features](#non-server-features)
 
+# History
+
+The history of this project essentially mirrors my development as a software engineer from the beginning of my hobby until now. If you don't care about the story, skip to the "Compatibility" or "Setup" sections below.
+
+I originally purchased PSO GC when I heard about PSUL, and wanted to play around with running homebrew on my GameCube. This pathway eventually led to [GCARS-CS](https://github.com/fuzziqersoftware/gcars-cs), but that's another story.
+
+<img align="left" src="static/s-khyps.png" /> After playing PSO for a while, both offline and online, I wrote a proxy called Khyps sometime in 2003. This was back in the days of the official Sega servers, where vulnerabilities weren't addressed in a timely manner or at all. It was common for malicious players using their own proxies or Action Replay codes (a story for another time) to send invalid commands that the servers would blindly forward, and cause the receiving clients to crash. These crashes were more than simply inconvenient; they could also corrupt your save data, destroying the hours of work you may have put into hunting items and leveling up your character.
+
+For a while it was essentially necessary to use a proxy to go online at all, so the proxy could block these invalid commands. Khyps was designed primarily with this function in mind, though it also implemented some convenient cheats, like the ability to give yourself or other players infinite HP and allow you to teleport to different places without using an in-game teleporter.
+
+<img align="left" src="static/s-khyller.png" /> After Khyps I took on the larger challenge of writing a server, which resulted in Khyller sometime in 2005. This was the first server of any type I had ever written. This project eventually evolved into a full-featured environment supporting all versions of the game that I had access to - at the time, PC, GC, and BB. (However, I suspect from reading the ancient source files that Khyller's BB support was very buggy.) As Khyller evolved, the code became increasingly cumbersome, littered with debugging filth that I never cleaned up and odd coding patterns I had picked up over the years. My understanding of the C++ language was woefully incomplete as well (as opposed to now, when it is still incomplete but not woefully so), which resulted in Khyller being essentially a C project that had a couple of classes in it.
+
+<img align="left" src="static/s-aeon.png" /> Sometime in 2006 or 2007, I abandoned Khyller and rebuilt the entire thing from scratch, resulting in Aeon. Aeon was substantially cleaner in code than Khyller but still fairly hard to work with, and it lacked a few of the more arcane features I had originally written (for example, the ability to convert any quest into a download quest). In addition, the code still had some stability problems... it turns out that Aeon's concurrency primitives were simply incorrect. I had derived the concept of a mutex myself, before taking any real computer engineering classes, but had implemented it incorrectly. I made the race window as small as possible, but Aeon would still randomly crash after running seemingly fine for a few days.
+
+At the time of its inception, Aeon was also called newserv, and you may find some beta releases floating around the Internet with filenames like `newserv-b3.zip`. I had released betas 1, 2, and 3 before I released the entire source of beta 5 and stopped working on the project when I went to college. This was around the time when I switched from writing software primarily on Windows to primarily on macOS and Linux, so Aeon beta 5 was the last server I wrote that specifically targeted Windows. (newserv, which you're looking at now, is a bit tedious to compile on Windows but does work.)
+
+<img align="left" src="static/s-newserv.png" /> After a long hiatus from PSO and much professional and personal development in my technical abilities, I was reminiscing sometime in October 2018 by reading my old code archives. Somehow inspired when I came across Aeon, I spent a weekend and a couple more evenings rewriting the entire project again, cleaning up ancient patterns I had used eleven years ago, replacing entire modules with simple STL containers, and eliminating even more support files in favor of configuration autodetection. The code is now suitably modern and stable, and I'm not embarrassed by its existence, as I am by Aeon beta 5's source code and my archive of Khyller (which, thankfully, no one else ever saw).
+
+## Other server projects
+
+Independently of this project, there are many other PSO servers out there. Those that I know of that are (or were) public are listed here in approximate chronological order:
+
+* (Early 2000s) **[Schtserv](https://schtserv.com/)**: The first public-access PSO server; written in Delphi by Schthack. Still active and popular as of this writing (early 2024). Schtserv is also the only other unofficial server to support all versions of PSO, including Episode 3.
+* (2005) **Khyller**: An early attempt of mine to support PSO PC, GC, and BB. See above for more details.
+* (2006) **Aeon**: My second attempt. Better than Khyller, but still unreliable.
+* (2008) **Tethealla**: A fairly extensive implementation of PSOBB, written in C by Sodaboy. The public version of Tethealla has been [officially disowned](https://www.pioneer2.net/community/threads/tethealla-server-forums-removal.26365/) (as it is now more than 15 years old), but closed-source development continues. [Ephinea](https://ephinea.pioneer2.net/), currently the most popular PSOBB server, is the continuation of this project. Several other modern PSOBB servers are forks of the initial public version of Tethealla as well.
+* (2008) **[Sylverant](https://sylverant.net/)** [(source)](https://sourceforge.net/projects/sylverant/): The second public-access PSO server; written in C by BlueCrab. Still active and popular as of this writing (early 2024).
+* (2015) **[Archon](https://github.com/dcrodman/archon)**: A PSOBB server written in Go by Drew Rodman.
+* (2015) **[Idola](https://github.com/HybridEidolon/idolapsoserv)**: A PSOBB server written in Rust by HybridEidolon. Functionality status unknown; the project has been archived.
+* (2017) **[Aselia](https://github.com/Solybum/Aselia)**: A PSOBB server written written in C# by Soly. It seems this was planned to be open-source at some point, but that has not (yet) happened.
+* (2018) **newserv**: This project right here.
+* (2019) **[Mechonis](https://gitlab.com/sora3087/mechonis)**: A PSOBB server with a microservice architecture written in TypeScript by TrueVision.
+* (2021) **[Phantasmal World](https://github.com/DaanVandenBosch/phantasmal-world)**: A set of PSO tools, including a web-based model viewer and quest builder, and a PSO server, written by Daan Vanden Bosch.
+* (2021) **[Elseware](http://git.sharnoth.com/jake/elseware)**: A PSOBB server written in Rust by Jake.
+
+## Developer information
+
+There is a lot of code in this project that could be useful as a reference. Some of the more notable files are:
+* **src/CommandFormats.hh**: Complete listing of all network commands used in all known versions of the game, and their formats
+* **src/DCSerialNumbers.hh/cc**: PSO DC serial number validation algorithm and serial number generator
+* **src/ItemData.hh**: Item format reference
+* **src/ItemCreator.hh/cc**: Reverse-engineered item generator from Episodes 1&2 (used for all versions)
+* **src/ItemParameterTable.hh**: Format of many structures in ItemPMT.prs
+* **src/Map.hh/cc**: Map file (.dat) structure and reverse-engineered Challenge Mode random enemy generation algorithm
+* **src/QuestScript.cc**: Complete listing of all quest opcodes on all versions, along with their arguments and behavior
+* **src/SaveFileFormats.hh**: Definitions of save file structures for all versions
+* **src/Episode3/DataIndexes.hh**: Episode 3 file structures, including card definition format and map/quest format
+* **system/item-tables/names-v4.json**: Names of all items, indexed by the first 3 bytes of data1
+
+## Using newserv in other projects
+
+There is a fair amount of code in this project that could potentially be useful to other projects. You are free to use code from newserv in your own open-source projects; the only condition is that the contents of the LICENSE file must be included in your project if you use code from newserv. Your project does not also have to use the MIT license; you can use any license you want.
+
+If you want to use parts of newserv in your project, there are two easy ways to do so with proper licensing:
+* If you're using a lot of code from newserv, you can put a copy of newserv's LICENSE file in your repository alongside your own license file, or include the contents of newserv's license in your own license file.
+* If you're only using a few files from newserv, you can copy and paste the contents of the LICENSE file into a comment at the beginning of each copied file.
+
 # Compatibility
 
-newserv supports several versions of PSO, including various development prototypes. Specifically:
-| Version        | Lobbies      | Games        | Proxy        |
-|----------------|--------------|--------------|--------------|
-| DC NTE         | Yes          | Yes          | No           |
-| DC 11/2000     | Yes          | Yes          | No           |
-| DC 12/2000     | Yes          | Yes          | Yes          |
-| DC 01/2001     | Yes          | Yes          | Yes          |
-| DC V1          | Yes          | Yes          | Yes          |
-| DC 08/2001     | Yes          | Yes          | Yes          |
-| DC V2          | Yes          | Yes          | Yes          |
-| PC NTE         | Yes (3)      | Yes          | No           |
-| PC             | Yes          | Yes          | Yes          |
-| GC Ep1&2 NTE   | Yes          | Yes          | Yes          |
-| GC Ep1&2       | Yes          | Yes          | Yes          |
-| GC Ep1&2 Plus  | Yes          | Yes          | Yes          |
-| GC Ep3 NTE     | Yes          | Yes (1)      | Yes          |
-| GC Ep3         | Yes          | Yes          | Yes          |
-| Xbox Ep1&2     | Yes          | Yes          | Yes          |
-| BB (vanilla)   | Yes          | Yes (2)      | Yes          |
-| BB (Tethealla) | Yes          | Yes (2)      | Yes          |
+newserv supports all known versions of PSO, including development prototypes. This table lists all versions that newserv supports. (NTE stands for Network Trial Edition; the GameCube beta versions were called Trial Edition instead, but we use the NTE abbreviation anyway for consistency.)
+
+| Version         | Lobbies  | Games    | Proxy    |
+|-----------------|----------|----------|----------|
+| DC NTE          | Yes      | Yes      | No       |
+| DC 11/2000      | Yes      | Yes      | No       |
+| DC 12/2000      | Yes      | Yes      | Yes      |
+| DC 01/2001      | Yes      | Yes      | Yes      |
+| DC V1           | Yes      | Yes      | Yes      |
+| DC 08/2001      | Yes      | Yes      | Yes      |
+| DC V2           | Yes      | Yes      | Yes      |
+| PC NTE          | Yes (3)  | Yes      | No       |
+| PC              | Yes      | Yes      | Yes      |
+| GC Ep1&2 NTE    | Yes      | Yes      | Yes      |
+| GC Ep1&2        | Yes      | Yes      | Yes      |
+| GC Ep1&2 Plus   | Yes      | Yes      | Yes      |
+| GC Ep3 NTE      | Yes      | Yes (1)  | Yes      |
+| GC Ep3          | Yes      | Yes      | Yes      |
+| Xbox Ep1&2 Beta | Yes      | Yes      | Yes      |
+| Xbox Ep1&2      | Yes      | Yes      | Yes      |
+| BB (vanilla)    | Yes      | Yes (2)  | Yes      |
+| BB (Tethealla)  | Yes      | Yes (2)  | Yes      |
 
 *Notes:*
-1. *Ep3 NTE battles are not well-tested; some things may not work. See notes/ep3-nte-differences.txt for a list of known differences between NTE and the final version. NTE and non-NTE players cannot battle each other.*
+1. *Episode 3 NTE battles are not well-tested; some things may not work. See notes/ep3-nte-differences.txt for a list of known differences between NTE and the final version. NTE and non-NTE players cannot battle each other.*
 2. *Some BB-specific features are not well-tested (for example, some quests that use rare commands may not work properly). Please submit a GitHub issue if you find something that doesn't work.*
 3. *This is the only version of PSO that doesn't have any way to identify the player's account - there is no serial number or username. For this reason, AllowUnregisteredUsers must be enabled in config.json to support PC NTE, and PC NTE players receive a random Guild Card number every time they connect. To prevent abuse, PC NTE support can be disabled in config.json.*
 
@@ -61,26 +127,30 @@ Currently newserv works on macOS, Windows, and Ubuntu Linux. It will likely work
 ### Windows/macOS
 
 1. Download the latest release-windows-amd64.zip or release-macos-arm64.zip file from the [releases page](https://github.com/fuzziqersoftware/newserv/releases).
-2. Extract the contents of the release folder to a location on your computer.
-3. Edit the config.example.json file in the system folder as needed, then rename it to config.json.
-4. If you plan to play Blue Burst on newserv, set up the patch directory. See [client patch directories](#client-patch-directories) for more information.
+2. Extract the contents of the archive to some location on your computer.
+3. (Optional) If you want to change any config options, go into the system/ folder, open config.json in a text editor, and edit it to your liking. There are comments in the file that describe what all the options do.
+4. (Optional) If you plan to play Blue Burst on newserv, set up the patch directory. See [client patch directories](#client-patch-directories) for details.
 5. Run the newserv executable.
 
 ### Linux
 
-There are currently no precompiled releases for Linux. To run newserv on Linux, see the "Building from source" section below.
+There are currently no precompiled releases for Linux. To run newserv on Linux, you'll have to build it from source - see the "Building from source" section below.
 
 ### Building from source
 
 1. Install the packages newserv depends on.
-    * If you're on Windows, install [Cygwin](https://www.cygwin.com/). While doing so, install the `cmake`, `gcc-core`, `gcc-g++`, `git`, `libevent2.1_7`, `make`, `libiconv-devel`, and `zlib` packages. Do the rest of these steps inside a Cygwin shell (not a Windows cmd shell or PowerShell).
+    * If you're on Windows, install [Cygwin](https://www.cygwin.com/). While doing so, install the `cmake`, `gcc-core`, `gcc-g++`, `git`, `libevent2.1_7`, `libevent-devel`, `make`, `libiconv-devel`, and `zlib` packages. Do the rest of these steps inside a Cygwin shell (not a Windows cmd shell or PowerShell).
     * If you're on macOS, run `brew install cmake libevent libiconv`.
     * If you're on Linux, run `sudo apt-get install cmake libevent-dev` (or use your Linux distribution's package manager).
 3. Build and install [phosg](https://github.com/fuzziqersoftware/phosg).
 4. Optionally, install [resource_dasm](https://github.com/fuzziqersoftware/resource_dasm). This will enable newserv to send memory patches and load DOL files on PSO GC clients. PSO GC clients can play PSO normally on newserv without this.
 5. Run `cmake . && make` in the newserv directory.
 
-After building newserv, edit system/config.example.json as needed and rename it to system/config.json, set up [client patch directories](#client-patch-directories) if you're planning to play Blue Burst, then run `./newserv` in newserv's directory.
+After building newserv, edit system/config.example.json as needed **and rename it to system/config.json** (note that this step is not necessary for the precompiled releases!), set up [client patch directories](#client-patch-directories) if you're planning to play Blue Burst, then run `./newserv` in newserv's directory.
+
+The server has an interactive shell which can be used to make changes, such as managing user accounts, updating the server's configuration, managing Episode 3 tournaments, and more. Type `help` and press Enter to see all the commands.
+
+On Linux and macOS, the server also responds to SIGUSR1 and SIGUSR2. SIGUSR1 does the equivalent of the shell's `reload config` command, which reloads config.json but not any dependent files (so quests, Episode 3 maps, etc. will not be reloaded). SIGUSR2 does the equivalent of the shell's `reload all` command, which reloads everything.
 
 To use newserv in other ways (e.g. for translating data), see the end of this document.
 
@@ -91,7 +161,10 @@ newserv implements a patch server for PSO PC and PSO BB game data. Any file or d
 For Blue Burst set up, the below is mandatory for a smooth experience:
 
 1. Browse to your chosen client's data directory.
-2. Copy all the map_*.dat files and the data.gsl file and place them in `system/patch-bb/data`
+2. Copy all the `map_*.dat` files, `unitxt_*` files and the `data.gsl` file and place them in `system/patch-bb/data`.
+3. If you're using game files from the Tethealla client, make a copy of `unitxt_j.prs` inside system/patch-bb/data and name it `unitxt_e.prs`. (If `unitxt_e.prs` already exists, replace it with the copied file.)
+
+If you don't have a BB client, or if you're using a Tethealla client from another source, Tethealla clients that are compatible with newserv can be found here: [English](https://web.archive.org/web/20240402011115/https://ragol.org/files/bb/TethVer12513_English.zip) / [Japanese](https://web.archive.org/web/20240402013127/https://ragol.org/files/bb/TethVer12513_Japanese.zip). These clients connect to 127.0.0.1 (localhost) automatically.
 
 For BB clients, newserv reads some files out of the patch data to implement game logic, so it's important that certain game files are synchronized between the server and the client. newserv contains defaults for these files in the system/maps/bb-v4 directory, but if these don't match the client's copies of the files, odd behavior will occur in games.
 
@@ -139,9 +212,24 @@ The version of PSO PC I have has the server addresses starting at offset 0x29CB3
 
 ### PSO GC on a real GameCube
 
-You can make PSO connect to newserv by setting its default gateway and DNS server addresses in network settings to newserv's address. newserv's DNS server must be running on port 53 and must be accessible to the GameCube.
+You can make PSO connect to newserv by setting the default gateway and DNS server addresses in the game's network settings to newserv's address. newserv's DNS server must be running on port 53 and must be accessible to the GameCube. If you're not playing PSO Plus or Episode III, this should be all you need to do, assuming you already set LocalAddress in config.json to your PC's private IP address.
 
-If you have PSO Plus or Episode III, it won't want to connect to a server on the same local network as the GameCube itself, as determined by the GameCube's IP address and subnet mask. In the old days, one way to get around this was to create a fake network adapter on the server (or use an existing real one) that has an IP address on a different subnet, tell the GameCube that the server is the default gateway (as above), and have the server reply to the DNS request with its non-local IP address. To do this with newserv, just set LocalAddress in the config file to a different interface. For example, if the GameCube is on the 192.168.0.x network and your other adapter has address 10.0.1.6, set newserv's LocalAddress to 10.0.1.6 and set PSO's DNS server and default gateway addresses to the server's 192.168.0.x address. This may not work on modern systems or on non-Windows machines - I haven't tested it in many years.
+If you have PSO Plus or Episode III, it won't want to connect to a server on the same local network as the GameCube itself, as determined by the GameCube's IP address and subnet mask. There are a couple of ways to get around this.
+
+Sodaboy described a fairly easy method, which is to forward the PSO and DNS ports in your router's configuration to your PC's private IP address (the PSO ports are in config.json, and are all TCP; the DNS port is 53 and is UDP). Then, set LocalAddress and ExternalAddress in config.json to your external IP address (from e.g. whatismyip.com). Most routers will let you connect to your public IP address even from within the local network, but the GameCube will think it's connecting to a different network, so it won't reject the connection. If you're concerned about security and don't want your server to be publicly accessible, you can use Windows Firewall or UFW on Linux block incoming connections on the ports you opened, except for connections from the IP addresses you specify.
+
+Another method is to use two network interfaces on the same PC, and tell the GameCube to connect to the one that appears to be on a different network. For example, if your GameCube is on the 10.0.0.x subnet and your PC's address is 10.0.0.5, you can create a fake network adapter on your PC (or use an existing real one) that has an IP address on a different subnet than the GameCube, such as 192.168.0.8. Then, in PSO's network config, set the default gateway and DNS server addresses to 192.168.0.8, and set LocalAddress in config.json to 192.168.0.8, and PSO should connect. This is what I did back in the old days when I primarily developed software on Windows, but I haven't tried it in many years.
+
+### PSO GC on a Wii or Wii U
+
+Using a Wii or Wii U to connect to newserv requires the Wii or vWii to be softmodded. How to do this is beyond the scope of this document.
+
+Nintendont includes BBA emulation and is compatible with all PSO GameCube versions except Episodes I&II Trial Edition. To use Nintendont, enable BBA emulation in Nintendont's settings and follow the instructions in the above section (PSO GC on a real GameCube).
+
+Devolution includes modem emulation and is compatible with all PSO GameCube versions including Episodes I&II Trial Edition. newserv can act as a PPP server, which Devolution can directly connect to. To do this:
+1. Enable the PPPRawListen option according to the comments in config.json.
+2. Start newserv.
+3. In the game's network settings, set the username and password to anything (they cannot be blank), and set the phone number to the number that newserv outputs to the console during startup. (It will be near the end of all the startup log messages.) If your Wii is on the same network as newserv, use the local number; otherwise, use the external number.
 
 ### PSO GC on Dolphin
 
@@ -149,9 +237,9 @@ If you're using the HLE BBA type, set the BBA's DNS server address to newserv's
 
 If you're using the TAP BBA type, you'll have to set PSO's network settings appropriately for your tap interface. Set the DNS server address in PSO's network settings to newserv's IP address.
 
-If you're using a version of Dolphin with tapserver support, you can make it connect to a newserv instance running on the same machine via the tapserver interface. You do not need to install or run tapserver. To do this:
-1. Set Dolphin's BBA type to tapserver (Config -> GameCube -> SP1).
-2. Enable newserv's IP stack simulator according to the comments in config.json and start newserv.
+If you're using the tapserver BBA or modem type, you can make it connect to a newserv instance running on the same machine via the tapserver interface. To do this:
+1. In the GameCube pane of the Config window, set the SP1 device to Broadband Adapter (tapserver) or Modem Adapter (tapserver).
+2. Set IPStackListen (for BBA) or PPPStackListen (for modem) according to the comments in config.json and start newserv.
 3. In PSO's network settings, enable DHCP ("Automatically obtain an IP address"), set DNS server address to "Automatic", and leave DHCP Hostname as "Not set". Leave the proxy server settings blank.
 4. Start an online game.
 
@@ -159,9 +247,12 @@ If you're using a version of Dolphin with tapserver support, you can make it con
 
 The PSO BB client has been modified and distributed in many different forms. newserv supports most, but not all, of the common distributions. Unlike other versions, it's important that the client and server have the same map files, so make sure to set up the patch directory based on the client you'll be using with newserv. (See the "Client patch directories" section for instructions on setting this up.)
 
-The original Japanese and US versions of PSO BB should work, but you'll have to modify your hosts file or edit psobb.exe to point to your newserv instance. The original versions are packed, so this is a more involved process than simply opening the executable in a hex editor and finding/replacing some strings.
+The original Japanese and US versions of PSO BB work with newserv (the last Japanese release can be found [here](https://archive.org/details/psobb_jp_setup_12511_20240109/)). To get them to connect to your server, do one of the following:
+* Use a drop-in patcher like [AzureFlare](https://github.com/Repflez/AzureFlare).
+* Modify your hosts file to redirect the client's destination address to localhost or your server's address.
+* Edit psobb.exe to point to your newserv instance. The original clients are packed with various versions of ASProtect, so this is a more involved process than simply opening the executable in a hex editor and finding/replacing some strings.
 
-Alternatively, you can use the Tethealla client (https://archive.org/details/psobb-tethealla-client); you can find the connection addresses starting at 0x56D724 in psobb.exe. Overwrite these addresses with your server's hostname or IP address, and you should be able to connect.
+Alternatively, you can use the Tethealla client ([English](https://web.archive.org/web/20240402011115/https://ragol.org/files/bb/TethVer12513_English.zip) or [Japanese](https://web.archive.org/web/20240402013127/https://ragol.org/files/bb/TethVer12513_Japanese.zip)). If the server is on the same PC as the client and you don't plan to have any external players, these Tethealla clients will automatically connect to the server without any modifications. This version of the client is not packed, and you can find the connection addresses starting at 0x56D724 in psobb.exe. Overwrite these addresses with your server's hostname or IP address, and you should be able to connect.
 
 ### Connecting external clients
 
@@ -171,6 +262,23 @@ For GC clients, you'll have to use newserv's built-in DNS server or set up your
 
 # Server feature configuration
 
+## User accounts
+
+By default, newserv does not require users to pre-register before playing; the server will instead automatically create an account the first time each player connects. These accounts have no special permissions. You can view, create, edit, and delete user accounts in the server's shell (run `help` in the shell to see how to do this).
+
+A license is a set of credentials that a player can use to log in. There are six types of licenses:
+* *DC NTE licenses* consist of a 16-character serial number and 16-character access key.
+* *DC licenses* consist of an 8-character hex serial number and an 8-character access key.
+* *PC licenses* are the same format as DC licenses, but are used for PC v2.
+* *GC licenses* consist of a 10-digit decimal serial number, a 12-character access key, and a password of up to 8 characters.
+* *XB licenses* consist of a gamertag of up to 16 characters, a 16-character hex user ID, and a 16-character hex account ID.
+* *BB licenses* consist of a username of up to 16 characters and a password of up to 16 characters.
+Each account may have multiple licenses. To add a license to an account, use `add-license` in the shell.
+
+On BB, character data is scoped to the license, but system and Guild Card data is scoped to the account. That is, an account with multiple BB licenses can have more than 4 characters (up to 4 per license), but they will all share the same team membership and Guild Card lists.
+
+You may want to give your account elevated privileges. To do so, run `update-account ACCOUNT-ID flags=root` (replacing ACCOUNT-ID with your actual account-id). You can also use update-account to edit other parts of the account; see the help text for more information.
+
 ## Installing quests
 
 newserv automatically finds quests in the subdirectories of the system/quests/ directory. To install your own quests, or to use quests you've saved using the proxy's Save Files option, just put them in one of the subdirectories there and name them appropriately. The subdirectories and their behaviors (e.g. in which game modes they should appear and for which PSO versions) is defined in the QuestCategories field in config.json.
@@ -238,7 +346,7 @@ There are five different available behaviors for item drops:
 
 In the `SERVER_PRIVATE` and `SERVER_DUPLICATE` modes, there is no incentive to pick up items before another player, since other players cannot pick up the items you see dropped from boxes and enemies. However, if you pick up an item and drop it later, it can then be seen and picked up by any player.
 
-The drop mode can be changed at any time during a game with the `$dropmode` chat command. If the mode is changed after some items have already been dropped, the existing items retain their visibility (that is, they still can't be picked up by other players since they were dropped before the mode was changed). You can configure which drop modes are used by default, and which modes players are allowed to choose, in config.json. See the comments above the AllowedDropModes and DefaultDropMode keys.
+The drop mode can be changed at any time during a game with the `$dropmode` chat command. If the mode is changed after some items have already been dropped, the existing items retain their visibility (that is, items dropped in private mode still can't be picked up by other players since they were dropped before the mode was changed). You can configure which drop modes are used by default, and which modes players are allowed to choose, in config.json. See the comments above the AllowedDropModes and DefaultDropMode keys.
 
 In the server drop modes, the item tables used to generate common items are in the `system/item-tables/ItemPT-*` files. (The V2 files are used for V1 as well.) The rare item tables are in the `rare-table-*.json` files. Unlike the original formats, it's possible to make each enemy drop multiple different rare items at different rates, though the default tables never do this.
 
@@ -252,6 +360,32 @@ All versions of PSO can see and interact with each other in the lobby. newserv a
 
 In V1/V2 cross-version play, when any of the server drop modes are used, the server uses the drop table corresponding to the version the game was created with. (For example, if a DC V1 player created the game, rare-table-v1.json will be used, even after V2 players join.)
 
+## Server-side saves
+
+newserv has the ability to save character data on the server side. For PSO BB, this is required of course, but this feature can also be used on other PSO versions.
+
+Each account has 4 BB character slots and 16 non-BB character file slots. The non-BB slots are independent of the BB slots, and can be accessed with the `$savechar <slot>` and `$loadchar <slot>` commands (slots are numbered 1 through 16). `$savechar` copies the character you're currently playing as and saves the data on the server, and `$loadchar` does the reverse, overwriting your current character with the data saved on the server. Note that you can load a character that was saved from a different version of PSO, which allows you to easily transfer characters between games. On v1 and v2, changes done by `$loadchar` will be undone if you join a game; to permanently save your changes, disconnect from the lobby after using the command.
+
+There is a third command, `$bbchar <username> <password> <slot>`, which behaves similarly to `$savechar` but writes the character data to a BB character slot in a different account instead (slots are numbered 1 through 4). This can be used to "upgrade" a character to BB from an earlier version.
+
+Exactly which data is saved and loaded depends on the game version:
+
+| Game                 | Inventory | Character | Options/chats | Quest flags | Bank | Battle/challenge |
+|----------------------|-----------|-----------|---------------|-------------|------|------------------|
+| PSO DC v1 prototypes | Yes       | Yes       | No            | No          | No   | N/A              |
+| PSO DC v1            | Yes       | Yes       | No            | No          | No   | N/A              |
+| PSO DC v2            | Yes       | Yes       | Yes           | Yes         | Yes  | Yes              |
+| PSO PC (v2)          | Yes       | Yes       | No            | No          | No   | Save only        |
+| PSO GC NTE           | Yes       | Yes       | Yes           | Yes         | Yes  | Yes              |
+| PSO GC (not Plus)    | Yes       | Yes       | Yes           | Yes         | Yes  | Yes              |
+| PSO GC Plus (1)      | Save only | Save only | No            | No          | No   | Save only        |
+| PSO GC Ep3 (1)       | No        | Save only | No            | No          | No   | Save only        |
+| PSO Xbox             | Yes       | Yes       | Yes           | Yes         | Yes  | Yes              |
+| PSO BB               | Yes       | Yes       | Yes           | Yes         | Yes  | Yes              |
+
+*Notes*:
+1. *If EnableSendFunctionCallQuestNumber is enabled in config.json, then $savechar and $loadchar can save and restore all character data on these versions, just like on GC non-Plus. Episode 3 characters exist in a separate namespace; that is, you can't use $savechar and $loadchar to convert an Ep3 character to non-Ep3, or vice versa.*
+
 ## Episode 3 features
 
 newserv supports many features unique to Episode 3:
@@ -283,35 +417,68 @@ Episode 3 state and game data is stored in the system/ep3 directory. The files i
 * card-text.mnr: Compressed card text archive. Generally only used for debugging.
 * card-text.mnrd: Decompressed card text archive; same format as TextCardE.bin. Generally only used for debugging.
 * com-decks.json: COM decks used in tournaments. The default decks in this file come from logs from Sega's servers, so the file doesn't include every COM deck Sega ever made - the rest are probably lost to time.
-* maps/: Online free battle and quest maps (.mnm/.bin/.mnmd/.bind files). newserv comes with all the original online and offline maps, including Story Mode quests. If you don't want the offline maps and quests to be playable online, delete the .bind files system/ep3/maps.
+* maps/: Online free battle and quest maps (.mnm/.bin/.mnmd/.bind files). newserv comes with the default online maps, as well as some fan-made variations and quests to help new players get up to speed.
 * maps-download/: Download maps and quests (.mnm/.bin/.mnmd/.bind files). There are two subcategories by default (download maps and Trial Edition download maps), but you can add more by editing QuestCategories in config.json. Categories that have flag 0x40 (Ep3 download) set are indexed from this directory; all others are indexed from system/quests/. Files in maps-download/ subdirectories have the same format as those in the maps/ directory, but should be named like `e###-gc3-LANGUAGE.EXT` (similar to how non-Episode 3 quests are named in the system/quests/ directory). If you want a map to be available for online play and for downloading, the file must exist in both maps/ and in a maps-download/ subdirectory (a symbolic link is acceptable).
+* maps-offline/: Offline map files. These are all the offline quests and free battle maps from the client, including some debugging/test maps that were inaccessible during normal play. To make them playable online, put the files in the maps/ directory.
 * tournament-state.json: State of all active tournaments. This file is automatically written when any tournament changes state for any reason (e.g. a tournament is created/started/deleted or a match is resolved).
 
-There is no public editor for Episode 3 maps and quests, but the format is described fairly thoroughly in src/Episode3/DataIndexes.hh (see the MapDefinition structure). You'll need to use `newserv decompress-prs ...` to decompress .bin or .mnm files before editing them, but you don't need to compress the files again to use them - just put the .bind or .mnmd file in the maps directory and newserv will make it available.
+There is no public editor for Episode 3 maps and quests, but the format is described fairly thoroughly in src/Episode3/DataIndexes.hh (see the MapDefinition structure). You'll need to use `newserv decompress-prs ...` to decompress a .bin or .mnm file before editing it, but you don't need to compress it again to use it - just put the .bind or .mnmd file in the maps directory and newserv will make it available.
 
-Like quests, Episode 3 card definitions, maps, and quests are cached in memory. If you've changed any of these files, you can run `reload ep3-data` in the interactive shell to make the changes take effect without restarting the server.
+Like quests, Episode 3 card definitions, maps, and quests are cached in memory. If you've changed any of these files, you can run `reload ep3-cards` or `reload ep3-maps` in the interactive shell to make the changes take effect without restarting the server.
 
 ## Memory patches, client functions, and DOL files
 
-Everything in this section requires resource_dasm to be installed, so newserv can use the assemblers and disassemblers from its libresource_file library. If resource_dasm is not installed, newserv will still build and run, but these features will not be available.
+*Everything in this section requires resource_dasm to be installed, so newserv can use the assemblers and disassemblers from its libresource_file library. If resource_dasm is not installed, newserv will still build and run, but these features will not be available.*
 
-In addition, these features are only supported for the following game versions:
-* PSO GameCube Episodes 1&2 Trial Edition
-* PSO GameCube Episodes 1&2 JP, USA, and EU but not Plus
-* PSO GameCube Episodes 1&2 Plus JP v1.4 but not v1.5
-* PSO GameCube Episode 3 Trial Edition
-* PSO GameCube Episode 3 JP
-* PSO GameCube Episode 3 USA (experimental; must be manually enabled in config.json)
-* PSO Xbox (all versions)
-* PSO BB
+You can put assembly files in the system/client-functions directory with filenames like PatchName.VERS.patch.s and they will appear in the Patches menu for clients that support client functions. Client functions are written in SH-4, PowerPC, or x86 assembly and are compiled when newserv is started. The assembly system's features are documented in the comments in system/client-functions/System/WriteMemory.ppc.s.
 
-*Note: newserv uses the shorter GameCube versioning convention, where discs labeled DOL-XXXX-0-0Y are version 1.Y. The PSO community seems to use the convention 1.0Y in some places instead, but these are the same version. For example, the version that newserv calls v1.4 is the same as v1.04, and is labeled DOL-GPOJ-0-04 on the underside of the disc.*
+The VERS token in client function filenames refers to the specific version of the game that the client function applies to. Some versions do not support receiving client functions at all. *Note: newserv uses the shorter GameCube versioning convention, where discs labeled DOL-XXXX-0-0Y are version 1.Y. The PSO community seems to use the convention 1.0Y in some places instead, but these are the same version. For example, the version that newserv calls v1.4 is the same as v1.04, and is labeled DOL-GPOJ-0-04 on the underside of the disc.*
 
-You can put memory patches in the system/client-functions directory with filenames like PatchName.patch.s and they will appear in the Patches menu for PSO GC, XB, and BB clients that support patching. Memory patches are written in PowerPC or x86 assembly and are compiled when newserv is started. The assembly system's features are documented in the comments in system/client-functions/WriteMemory.ppc.s.
+The specific versions are:
 
-newserv comes with a set of patches for GC Episodes 1&2 based on AR codes originally made by Ralf at GC-Forever. Many of them were originally posted in [this thread](https://www.gc-forever.com/forums/viewtopic.php?f=38&t=2050).
+| Game              | VERS | Architecture  |
+|-------------------|------|---------------|
+| PSO DC NTE        | 1OJ1 | Not supported |
+| PSO DC 11/2000    | 1OJ2 | Not supported |
+| PSO DC 12/2000    | 1OJ3 | Not supported |
+| PSO DC 01/2001    | 1OJ4 | Not supported |
+| PSO DC v1 JP      | 1OJF | Not supported |
+| PSO DC v1 US      | 1OEF | Not supported |
+| PSO DC v1 EU      | 1OPF | Not supported |
+| PSO DC 08/2001    | 2OJ5 | SH-4          |
+| PSO DC v2 JP      | 2OJF | SH-4          |
+| PSO DC v2 US      | 2OEF | SH-4          |
+| PSO DC v2 EU      | 2OPF | SH-4          |
+| PSO PC (v2)       | 2OJW | Not supported |
+| PSO GC NTE        | 3OJT | PowerPC       |
+| PSO GC v1.2 JP    | 3OJ2 | PowerPC       |
+| PSO GC v1.3 JP    | 3OJ3 | PowerPC       |
+| PSO GC v1.4 JP    | 3OJ4 | PowerPC       |
+| PSO GC v1.5 JP    | 3OJ5 | PowerPC (1)   |
+| PSO GC v1.0 US    | 3OE0 | PowerPC       |
+| PSO GC v1.1 US    | 3OE1 | PowerPC       |
+| PSO GC v1.2 US    | 3OE2 | PowerPC (1)   |
+| PSO GC v1.0 EU    | 3OP0 | PowerPC       |
+| PSO GC Ep3 NTE    | 3SJT | PowerPC       |
+| PSO GC Ep3 JP     | 3SJ0 | PowerPC       |
+| PSO GC Ep3 US     | 3SE0 | PowerPC (1)   |
+| PSO GC Ep3 EU     | 3SP0 | PowerPC (1)   |
+| PSO Xbox Beta     | 4OJB | x86           |
+| PSO Xbox JP Disc  | 4OJD | x86           |
+| PSO Xbox JP TU    | 4OJU | x86           |
+| PSO Xbox US Disc  | 4OED | x86           |
+| PSO Xbox US TU    | 4OEU | x86           |
+| PSO Xbox EU Disc  | 4OPD | x86           |
+| PSO Xbox EU TU    | 4OPU | x86           |
+| PSO BB JP 1.25.13 | 59NL | x86           |
+| PSO BB Tethealla  | 59NL | x86           |
 
-You can also put DOL files in the system/dol directory, and they will appear in the Programs menu for GC clients. Selecting a DOL file there will load the file into the GameCube's memory and run it, just like the old homebrew loaders (PSUL and PSOload) did. For this to work, ReadMemoryWord.ppc.s, WriteMemory.ppc.s, and RunDOL.ppc.s must be present in the system/client-functions directory. This has been tested on Dolphin but not on a real GameCube, so results may vary.
+*Notes:*
+1. *Client functions are only supported on these versions if EnableSendFunctionCallQuestNumbers is set in config.json. See the comments there for more information.*
+
+newserv comes with a set of patches for many of the above versions, based on AR codes originally made by Ralf at GC-Forever and Aleron Ives. Many of them were originally posted in [this thread](https://www.gc-forever.com/forums/viewtopic.php?f=38&t=2050).
+
+You can also put DOL files in the system/dol directory, and they will appear in the Programs menu for GC clients. Selecting a DOL file there will load the file into the GameCube's memory and run it, just like the old homebrew loaders (PSUL and PSOload) did. For this to work, ReadMemoryWord.ppc.s, WriteMemory.ppc.s, and RunDOL.ppc.s must be present in the system/client-functions/System directory. This has been tested on Dolphin but not on a real GameCube, so results may vary.
 
 Like other kinds of data, functions and DOL files are cached in memory. If you've changed any of these files, you can run `reload functions` or `reload dol-files` in the interactive shell to make the changes take effect without restarting the server.
 
@@ -331,10 +498,10 @@ There are many options available when starting a proxy session. All options are
 * **Chat commands**: enables chat commands in the proxy session (on by default).
 * **Chat filter**: enables escape sequences in chat messages and info board (on by default).
 * **Player notifications**: shows a message when any player joins or leaves the game or lobby you're in.
-* **Block pings**: blocks automatic pings sent by the client, and responds to ping commands from the server automatically. This works around a bug in Sylverant's login server.
+* **Block pings**: blocks automatic pings sent by the client, and responds to ping commands from the server automatically.
 * **Infinite HP**: automatically heals you whenever you get hit. An attack that kills you in one hit will still kill you, however.
 * **Infinite TP**: automatically restores your TP whenever you use any technique.
-* **Switch assist**: attempts to unlock doors that require two players in a one-player game.
+* **Switch assist**: unlocks doors that require two or four players in a one-player game, when you step on one of the switches.
 * **Infinite Meseta** (Episode 3 only): gives you 1,000,000 Meseta, regardless of the value sent by the remote server.
 * **Block events**: disables holiday events sent by the remote server.
 * **Block patches**: prevents any B2 (patch) commands from reaching the client.
@@ -359,91 +526,141 @@ newserv supports a variety of commands players can use by chatting in-game. Any
 Some commands only work on the game server and not on the proxy server. The chat commands are:
 
 * Information commands
-    * `$li`: Shows basic information about the lobby or game you're in. If you're on the proxy server, shows information about your connection instead (remote Guild Card number, client ID, etc.).
-    * `$si` (game server only): Shows basic information about the server.
-    * `$ping`: Shows round-trip ping time from the server to you. On the proxy server, shows the ping time from you to the proxy and from the proxy to the server.
-    * `$matcount` (game server only): Shows how many of each type of material you've used.
-    * `$itemnotifs <mode>`: Enables item drop notification messages. The modes are `off`, `rare`, and `on`, which should be self-explanatory. If the game has private drops enabled, you will only see a notification if the dropped item is visible to you; you won't be notified of other players' rare drops.
-    * `$what` (game server only): Shows the type, name, and stats of the nearest item on the ground.
-    * `$where` (game server only): Shows your current floor number and coordinates. Mainly useful for debugging.
+    * `$li`: Show basic information about the lobby or game you're in. If you're on the proxy server, show information about your connection instead (remote Guild Card number, client ID, etc.).
+    * `$si` (game server only): Show basic information about the server.
+    * `$ping`: Show round-trip ping time from the server to you. On the proxy server, show the ping time from you to the proxy and from the proxy to the server.
+    * `$matcount` (game server only): Show how many of each type of material you've used.
+    * `$killcount` (game server only): Show the kill count on your currently-equipped weapon. If you're in a game and not on BB, the value is only accurate at the time the item enters the game.
+    * `$itemnotifs <mode>`: Enable item drop notification messages. If the game has private drops enabled, you will only see a notification if the dropped item is visible to you; you won't be notified of other players' drops. The modes are:
+        * `off`: No notifications are shown.
+        * `rare`: You are notified when a rare item drops.
+        * `on`: You are notified when any item drops, except Meseta.
+        * `every`: You are notified when any item drops, including Meseta.
+    * `$announcerares`: Enable or disable announcements for your rare item finds. This determines whether rare items you find will be announced to the game and server, not whether you will see announcements for others finding rare items.
+    * `$what` (game server only): Show the type, name, and stats of the nearest item on the ground.
+    * `$where` (game server only): Show your current floor number and coordinates. Mainly useful for debugging.
+    * `$qfread <field-name>` (game server only): Show the value of a quest counter in your player data. The field names are defined in config.json.
 
 * Debugging commands
-    * `$debug` (game server only): Enable or disable debug. You need the DEBUG permission in your user license to use this command. Enabling debug does a few things:
+    * `$debug` (game server only): Enable or disable debug. You need the DEBUG flag in your user account to use this command. Enabling debug does several things:
         * You'll see in-game messages from the server when you take certain actions, like killing an enemy in BB.
         * You'll see the rare seed value and floor variations when you join a game.
-        * You'll be placed into the highest available slot in lobbies and games instead of the lowest, unless you're joining a BB solo-mode game.
-        * You'll be able to join games with any PSO version, not only those for which crossplay is normally supported. Be prepared for client crashes and other client-side brokenness if you do this. Please do not submit any issues for broken behaviors in crossplay, unless the situation is explicitly supported (see the "Cross-version play" section above).
+        * You'll be placed into the last available slot in lobbies and games instead of the first, unless you're joining a BB solo-mode game.
+        * You'll be able to join games with any PSO version, not only those for which crossplay is normally supported. Be prepared for client crashes and other client-side brokenness if you do this. Do not submit any issues for broken behaviors in crossplay, unless the situation is explicitly supported (see the "Cross-version play" section above).
         * The rest of the commands in this section are enabled on the game server. (They are always enabled on the proxy server.)
-    * `$quest <number>` (game server only): Load a quest by quest number. Can be used to load battle or challenge quests with only one player present.
+    * `$quest <number>` (game server only): Load a quest by quest number. Can be used to load battle or challenge quests with only one player present. Debug is not required to be enabled if the specified quest has the AllowStartFromChatCommand field set in its metadata file.
     * `$qcall <function-id>`: Call a quest function on your client.
-    * `$qcheck <flag-num>` (game server only): Show the value of a quest flag. This command can be used without debug mode enabled.
-    * `$qset <flag-num>` or `$qclear <flag-num>`: Set or clear a quest flag for everyone in the game.
-    * `$qgread <flag-num>` (game server only): Get the value of a quest counter ("global flag"). This command can be used without debug mode enabled.
+    * `$qcheck <flag-num>` (game server only): Show the value of a quest flag. This command can be used without debug mode enabled. If you're in a game, show the value of the flag in that game; if you're in the lobby, show the saved value of that quest flag for your character (BB only).
+    * `$qset <flag-num>` or `$qclear <flag-num>`: Set or clear a quest flag for everyone in the game. If you're in the lobby and on BB, set or clear the saved value of a quest flag in your character file.
+    * `$qgread <flag-num>` (game server only): Show the value of a quest counter ("global flag"). This command can be used without debug mode enabled.
     * `$qgwrite <flag-num> <value>` (game server only): Set the value of a quest counter ("global flag") for yourself.
     * `$qsync <reg-num> <value>`: Set a quest register's value for yourself only. `<reg-num>` should be either rXX (e.g. r60) or fXX (e.g. f60); if the latter, `<value>` is parsed as a floating-point value instead of as an integer.
     * `$qsyncall <reg-num> <value>`: Set a quest register's value for everyone in the game. `<reg-num>` should be either rXX (e.g. r60) or fXX (e.g. f60); if the latter, `<value>` is parsed as a floating-point value instead of as an integer.
+    * `$swset [floor] <flag-num>` and `$swclear [floor] <flag-num>`: Set or clear a switch flag. If floor is not given, sets or clears the flag on your current floor.
+    * `$swsetall`: Set all switch flags on your current floor. This unlocks all doors, disables all laser fences, triggers all light/poison switches, etc.
     * `$gc` (game server only): Send your own Guild Card to yourself.
     * `$sc <data>`: Send a command to yourself.
-    * `$ss <data>` (proxy server only): Send a command to the remote server.
+    * `$ss <data>`: Send a command to the remote server (if in a proxy session) or to the game server.
+    * `$sb <data>`: Send a command to yourself, and to the remote server or game server.
     * `$meseta <amount>` (game server only; Episode 3 only): Add the given amount to your Meseta total.
     * `$auction` (Episode 3 only): Bring up the CARD Auction menu, regardless of how many players are in the game or if you have a VIP card.
     * `$ep3battledebug` (game server only; Episode 3 only): Enable or disable TCard00_Select. If enabled, the game will enter the debug menu when you start a battle.
 
 * Personal state commands
-    * `$arrow <color-id>`: Changes your lobby arrow color.
-    * `$secid <section-id>`: Sets your override section ID. After running this command, any games you create will use your override section ID for rare drops instead of your character's actual section ID. To revert to your actual section id, run `$secid` with no name after it. On the proxy server, this will not work if the remote server controls item drops (e.g. on BB, or on Schtserv with server drops enabled). If the server does not allow cheat mode anywhere (that is, "CheatModeBehavior" is "Off" in config.json), this command does nothing.
-    * `$rand <seed>`: Sets your override random seed (specified as a 32-bit hex value). This will make any games you create use the given seed for rare enemies. This also makes item drops deterministic in Blue Burst games hosted by newserv. On the proxy server, this command can cause desyncs with other players in the same game, since they will not see the overridden random seed. To remove the override, run `$rand` with no arguments. If the server does not allow cheat mode anywhere (that is, "CheatModeBehavior" is "Off" in config.json), this command does nothing.
-    * `$ln [name-or-type]`: Sets the lobby number. Visible only to you. This command exists because some non-lobby maps can be loaded as lobbies with invalid lobby numbers. See the "GC lobby types" and "Ep3 lobby types" entries in the information menu for acceptable values here. Note that non-lobby maps do not have a lobby counter, so there's no way to exit the lobby without using either `$ln` again or `$exit`. On the game server, `$ln` reloads the lobby immediately; on the proxy server, it doesn't take effect until you load another lobby yourself (which means you'll like have to use `$exit` to escape). Run this command with no argument to return to the default lobby.
-    * `$swa`: Enables or disables switch assist. When enabled, the server will attempt to automatically unlock two-player doors in non-quest games if you step on both switches sequentially.
-    * `$exit`: If you're in a lobby, sends you to the main menu (which ends your proxy session, if you're in one). If you're in a game or spectator team, sends you to the lobby (but does not end your proxy session if you're in one). Does nothing if you're in a non-Episode 3 game and no quest is in progress.
+    * `$arrow <color-id>`: Change your lobby arrow color.
+    * `$secid <section-id>`: Set your override section ID. After running this command, any games you create will use your override section ID for rare drops instead of your character's actual section ID. If you're in a game and you are the leader of the game, this also immediately changes the item tables used by the server when creating items. To revert to your actual section id, run `$secid` with no name after it. On the proxy server, this will not work if the remote server controls item drops (e.g. on BB, or on Schtserv with server drops enabled). If the server does not allow cheat mode anywhere (that is, "CheatModeBehavior" is "Off" in config.json), this command does nothing.
+    * `$battle` (game server only; DC v1 only): After using this command, the next game you create will be in battle mode. (A chat command is required for this because DCv1 doesn't allow this natively.) On DCv1, the battle quests are not available, but free-roam is.
+    * `$rand <seed>`: Set your override random seed (specified as a 32-bit hex value). This will make any games you create use the given seed for rare enemies. This also makes item drops deterministic in Blue Burst games hosted by newserv. On the proxy server, this command can cause desyncs with other players in the same game, since they will not see the overridden random seed. To remove the override, run `$rand` with no arguments. If the server does not allow cheat mode anywhere (that is, "CheatModeBehavior" is "Off" in config.json), this command does nothing.
+    * `$ln [name-or-type]`: Set the lobby number. Visible only to you. This command exists because some non-lobby maps can be loaded as lobbies with invalid lobby numbers. See the "GC lobby types" and "Ep3 lobby types" entries in the information menu for acceptable values here. Note that non-lobby maps do not have a lobby counter, so there's no way to exit the lobby without using either `$ln` again or `$exit`. On the game server, `$ln` reloads the lobby immediately; on the proxy server, it doesn't take effect until you load another lobby yourself (which means you'll like have to use `$exit` to escape). Run this command with no argument to return to the default lobby.
+    * `$swa`: Enable or disable switch assist. When enabled, the server will unlock two-player and four-player doors in non-quest games when you step on any of the required switches.
+    * `$exit`: If you're in a lobby, send you to the main menu (which ends your proxy session, if you're in one). If you're in a game or spectator team, send you to the lobby (but does not end your proxy session if you're in one). Does nothing if you're in a non-Episode 3 game and no quest is in progress.
     * `$patch <name>`: Run a patch on your client. `<name>` must exactly match the name of a patch on the server.
 
 * Character data commands (game server only)
-    * `$savechar <slot>`: Saves your current character data on the server in the specified slot (each serial number has 4 slots, numbered 1-4). These slots are separate from BB character slots; using this command does not affect BB characters.
-    * `$loadchar <slot>` (v1 and v2 only): Loads your character data from the specified slot. The changes will be undone if you join a game - to save your changes, disconnect from the lobby.
-    * `$bbchar <username> <password> <slot>`: Use this command when playing on a non-BB version of PSO. If the username and password are correct, this command converts your current character to BB format and saves it on the server in the given slot (1-4). Any character already in that slot is overwritten. (This command is similar to `$savechar`, except it overwrites a BB character slot, and can transfer characters across accounts.) Note that the character's chat data, quick menu config, and bank contents are not copied, since there is no way for the server to request those types of data.
-    * `$edit <stat> <value>`: Modifies your character data. If you are on V3 (GameCube/Xbox), this command does nothing. If you are on V1 or V2 (DC or PC, not BB), your changes will be undone if you join a game - to save your changes, disconnect from the lobby. If cheats are allowed on the server, `<stat>` can be any of `atp`, `mst`, `evp`, `hp`, `dfp`, `ata`, `lck`, `meseta`, `exp`, `level`, `namecolor`, `secid`, `name`, `npc`, or `tech`. If cheats are not allowed, only `namecolor`, `name`, and `npc` can be used.
+    * `$savechar <slot>`: Save your current character data on the server in the specified slot. See the "Server-side saves" section for more details.
+    * `$loadchar <slot>`: Save your current character data on the server in the specified slot. See the "Server-side saves" section for more details.
+    * `$bbchar <username> <password> <slot>`: Save your current character data on the server in a different account's BB character slots. See the "Server-side saves" section for more details.
+    * `$edit <stat> <value>`: Modify your character data. See "Using $edit" below for details.
 
 * Blue Burst player commands (game server only)
-    * `$bank [number]`: Switches your current bank, so you can access your other character's banks (if `number` is 1-4) or your shared account bank (if `number` is 0). If `number` is not given, switches back to your current character's bank.
-    * `$save`: Saves your character, system, and Guild Card data immediately. (By default, your character is saved every 60 seconds while online, and your account and Guild Card data are saved whenever they change.)
+    * `$bank [number]`: Switch your current bank, so you can access your other character's banks (if `number` is 1-4) or your shared account bank (if `number` is 0). If `number` is not given, switch back to your current character's bank.
+    * `$save`: Save your character, system, and Guild Card data immediately. (By default, your character is saved every 60 seconds while online, and your account and Guild Card data are saved whenever they change.)
 
 * Game state commands (game server only)
-    * `$maxlevel <level>`: Sets the maximum level for players to join the current game. (This only applies when joining; if a player joins and then levels up past this level during the game, they are not kicked out, but won't be able to rejoin if they leave.)
-    * `$minlevel <level>`: Sets the minimum level for players to join the current game.
-    * `$password <password>`: Sets the game's join password. To unlock the game, run `$password` with nothing after it.
-    * `$dropmode [mode]`: Changes the way item drops behave in the current game. `mode` can be `none`, `client`, `shared`, `private`, or `duplicate`. If `mode` is not given, tells you the current drop mode without changing it. See the "Item tables and drop modes" section for more information.
-    * `$persist`: Enable or disable persistence for the current game. When persistence is on, the game will not be deleted when the last player leaves. The state of enemies and objects on the map will be reset when the last player leaves, but dropped items will not be deleted. If the game is empty for too long (15 minutes by default), it is then deleted.
+    * `$maxlevel <level>`: Set the maximum level for players to join the current game. (This only applies when joining; if a player joins and then levels up past this level during the game, they are not kicked out, but won't be able to rejoin if they leave.)
+    * `$minlevel <level>`: Set the minimum level for players to join the current game.
+    * `$password <password>`: Set the game's join password. To unlock the game, run `$password` with nothing after it.
+    * `$dropmode [mode]`: Change the way item drops behave in the current game. `mode` can be `none`, `client`, `shared`, `private`, or `duplicate`. If `mode` is not given, tells you the current drop mode without changing it. See the "Item tables and drop modes" section for more information.
+    * `$persist`: Enable or disable persistence for the current game. When persistence is on, the game will not be deleted when the last player leaves. The states of enemies, objects, and switches will be saved, and items left on the floor will not be deleted. (But if you're in the private or duplicate drop mode, items dropped by enemies are deleted - to make sure a certain item won't be deleted, you can pick it up and drop it again.) If the game is empty for too long (15 minutes by default), it is then deleted.
 
 * Episode 3 commands (game server only)
-    * `$spec`: Toggles the allow spectators flag for Episode 3 games. If any players are spectating when this flag is disabled, they will be sent back to the lobby.
-    * `$inftime`: Toggles infinite-time mode. Must be used before starting a battle. If infinite-time mode is enabled, the overall and per-phase time limits will be disabled regardless of the values chosen during battle setup. After completing a battle, infinite-time mode is reset to the server's default value (which can be set in Episode3BehaviorFlags in config.json).
-    * `$defrange <min>-<max>`: Sets the DEF dice range for the next battle. If this is used, the dice range set during battle rules setup will apply only to ATK dice; DEF dice will use this range instead. Assist cards and other dice effects will still apply. Dice exchange also still applies if it is enabled.
-    * `$stat <what>`: Shows a statistic about your player or team in the current battle. `<what>` can be `duration`, `fcs-destroyed`, `cards-destroyed`, `damage-given`, `damage-taken`, `opp-cards-destroyed`, `own-cards-destroyed`, `move-distance`, `cards-set`, `fcs-set`, `attack-actions-set`, `techs-set`, `assists-set`, `defenses-self`, `defenses-ally`, `cards-drawn`, `max-attack-damage`, `max-combo`, `attacks-given`, `attacks-taken`, `sc-damage`, `damage-defended`, or `rank`.
-    * `$surrender`: Causes your team to immediately lose the current battle.
-    * `$saverec <name>`: Saves the recording of the last battle.
-    * `$playrec <name>`: Plays a battle recording. This command creates a spectator team and replays the specified battle log within it. There is a bug in Dolphin that makes use of this command unstable in emulation (see the "Battle records" section above).
+    * `$spec`: Toggle the allow spectators flag for Episode 3 games. If any players are spectating when this flag is disabled, they are sent back to the lobby.
+    * `$inftime`: Toggle infinite-time mode. Must be used before starting a battle. If infinite-time mode is on, the overall and per-phase time limits will be disabled regardless of the values chosen during battle rules setup. After completing a battle, infinite-time mode is reset to the server's default value (which can be set in Episode3BehaviorFlags in config.json).
+    * `$dicerange [d:L-H] [1:L-H] [a1:L-H] [d1:L-H]`: Set override dice ranges for the next battle. The min and max dice values from the rules setup menu always apply to the ATK dice, but you can specify a different range for the DEF dice with `d:2-4` (for example). The `1:` override applies to the 1-player team in a 2v1 game (so you would set the 2-player team's desired dice range in the rules menu). You can also specify the 1-player team's ATK and DEF ranges separately with the `a1:` and `d1:` overrides. Note that these ranges will only be used if the chosen map or quest does not override them.
+    * `$stat <what>`: Show a statistic about your player or team in the current battle. `<what>` can be `duration`, `fcs-destroyed`, `cards-destroyed`, `damage-given`, `damage-taken`, `opp-cards-destroyed`, `own-cards-destroyed`, `move-distance`, `cards-set`, `fcs-set`, `attack-actions-set`, `techs-set`, `assists-set`, `defenses-self`, `defenses-ally`, `cards-drawn`, `max-attack-damage`, `max-combo`, `attacks-given`, `attacks-taken`, `sc-damage`, `damage-defended`, or `rank`.
+    * `$surrender`: Cause your team to immediately lose the current battle. If your story character is already defeated, you can't surrender - only your teammate can.
+    * `$saverec <name>`: Save the recording of the last battle.
+    * `$playrec <name>`: Play a battle recording. This command creates a spectator team immediately but the replay does not start automatically, to give other players a chance to join. To start the battle replay within the spectator team, run `$playrec` again (with no name). There is a bug in Dolphin that makes this command unstable in emulation (see the "Battle records" section above).
 
 * Cheat mode commands
-    * `$cheat` (game server only): Enables or disables cheat mode for the current game. All other cheat mode commands do nothing if cheat mode is disabled. By default, cheat mode is off in new games but can be enabled; there is an option in config.json that allows you to disable cheat mode entirely, or set it to on by default in new games. Cheat mode is always enabled on the proxy server, unless cheat mode is disabled on the entire server.
-    * `$infhp` / `$inftp`: Enables or disables infinite HP or TP mode. Applies to only you. In infinite HP mode, one-hit KO attacks will still kill you. On V1 and V2, infinite HP also automatically cures status ailments.
-    * `$warpme <floor-id>` (or `$warp <floor-id>`): Warps yourself to the given floor.
-    * `$warpall <floor-id>`: Warps everyone in the game to the given floor. You must be the leader to use this command, unless you're on the proxy server.
-    * `$next`: Warps yourself to the next floor.
+    * `$cheat` (game server only): Enable or disable cheat mode for the current game. All other cheat mode commands do nothing if cheat mode is disabled. By default, cheat mode is off in new games but can be enabled; there is an option in config.json that allows you to disable cheat mode entirely, or set it to on by default in new games. Cheat mode is always enabled on the proxy server, unless cheat mode is disabled on the entire server.
+    * `$infhp`: Enable or disable infinite HP mode. Applies to only you; does not affect other players. When enabled, one-hit KO attacks will still kill you, but on most versions of the game (not DCv1, GC US 1.2, or GC JP 1.5), the server will automatically revive you if you die. On all versions except GC US 1.2 and GC JP 1.5, infinite HP also automatically cures status ailments.
+    * `$inftp`: Enable or disable infinite TP mode. Applies to only you; does not affect other players.
+    * `$warpme <floor-id>` (or `$warp <floor-id>`): Warp yourself to the given floor.
+    * `$warpall <floor-id>`: Warp everyone in the game to the given floor. You must be the leader to use this command, unless you're on the proxy server.
+    * `$next`: Warp yourself to the next floor.
     * `$item <desc>` (or `$i <desc>`): Create an item. `desc` may be a description of the item (e.g. "Hell Saber +5 0/10/25/0/10") or a string of hex data specifying the item code. Item codes are 16 hex bytes; at least 2 bytes must be specified, and all unspecified bytes are zeroes. If you are on the proxy server, you must not be using Blue Burst for this command to work. On the game server, this command works for all versions.
     * `$unset <index>` (game server only): In an Episode 3 battle, removes one of your set cards from the field. `<index>` is the index of the set card as it appears on your screen - 1 is the card next to your SC's icon, 2 is the card to the right of 1, etc. This does not cause a Hunters-side SC to lose HP, as they normally do when their items are destroyed.
+    * `$dropmode [mode]` (proxy server): Change the way item drops behave in the current game, if you are not on BB. Unlike the game server version of this command, using this on the proxy server requires cheats to be enabled. This works by intercepting the drop requests sent to and from the leader. (So, if you are the leader and not using server drop mode on the remote server, it affects the entire game; otherwise, it affects only items generated by your actions.) `mode` can be `none` (no drops), `default` (normal drops), or `proxy` (use newserv's drop tables instead of the remote server's). If `mode` is not given, tells you the current drop mode without changing it.
 
-* Configuration commands
-    * `$event <event>`: Sets the current holiday event in the current lobby. Holiday events are documented in the "Using $event" item in the information menu. If you're on the proxy server, this applies to all lobbies and games you join, but only you will see the new event - other players will not.
-    * `$allevent <event>` (game server only): Sets the current holiday event in all lobbies.
-    * `$song <song-id>` (Episode 3 only): Plays a specific song in the current lobby.
+* Aesthetic commands
+    * `$event <event>`: Set the current holiday event in the current lobby. Holiday events are documented in the "Using $event" item in the information menu. If you're on the proxy server, this applies to all lobbies and games you join, but only you will see the new event - other players will not.
+    * `$allevent <event>` (game server only): Set the current holiday event in all lobbies.
+    * `$song <song-id>` (Episode 3 only): Play a specific song in the current lobby.
 
 * Administration commands (game server only)
-    * `$ann <message>`: Sends an announcement message. The message text is sent to all players in all games and lobbies.
-    * `$ax <message>`: Sends a message to the server's terminal. This cannot be used to run server shell commands; it only prints text to stderr.
-    * `$silence <identifier>`: Silences a player (remove their ability to chat) or unsilences a player. The identifier may be the player's name or Guild Card number.
-    * `$kick <identifier>`: Disconnects a player. The identifier may be the player's name or Guild Card number.
-    * `$ban <identifier>`: Bans a player. The identifier may be the player's name or Guild Card number.
+    * `$ann <message>`: Send an announcement message. The message is sent as temporary on-screen text to all players in all games and lobbies. On BB, the message appears in the scrolling top bar.
+    * `$ann!`, `$ann?`, `$ann?!`: Same as `$ann`, but with `?`, omits the sender's name, and with `!`, sends the message as a Simple Mail message instead of on-screen text.
+    * `$ax <message>`: Send a message to the server's terminal. This cannot be used to run server shell commands; it only prints text to stderr.
+    * `$silence <identifier>`: Silence a player (remove their ability to chat) or unsilence a player. The identifier may be the player's name or Guild Card number.
+    * `$kick <identifier>`: Disconnect a player. The identifier may be the player's name or Guild Card number.
+    * `$ban <duration> <identifier>`: Ban a player. The duration should be of the form `10m` (minutes), `10h` (hours), `10d` (days), `10w` (weeks), `10M` (months), or `10y` (years). (Numbers other than 10 may be used, of course.) As with `$kick`, the identifier may be the player's name or Guild Card number.
+
+### Using $edit
+
+The $edit command modifies your character data. This command doesn't work on V3 (GameCube/Xbox). If you are on V1 or V2 (DC or PC, not BB), your changes will be undone if you join a game - to save your changes, disconnect from the lobby.
+
+Some subcommands are always available. They are:
+* `$edit mat reset power`: Clear your usage of power materials (BB only)
+* `$edit mat reset mind`: Clear your usage of mind materials (BB only)
+* `$edit mat reset evade`: Clear your usage of evade materials (BB only)
+* `$edit mat reset def`: Clear your usage of def materials (BB only)
+* `$edit mat reset luck`: Clear your usage of luck materials (BB only)
+* `$edit mat reset hp`: Clear your usage of HP materials (BB only)
+* `$edit mat reset tp`: Clear your usage of TP materials (BB only)
+* `$edit mat reset all`: Clear your usage of all materials except HP and TP (BB only)
+* `$edit mat reset every`: Clear your usage of all materials including HP and TP (BB only)
+* `$edit namecolor AARRGGBB`: Set your name color (AARRGGBB specified in hex)
+* `$edit language L`: Set your language (Generally only useful on BB; values for L: J = Japanese, E = English, G = German, F = French, S = Spanish, B = Simplified Chinese, T = Traditional Chinese, K = Korean)
+* `$edit name NAME`: Set your character name
+* `$edit npc NPC-NAME`: Set or remove an NPC skin on your character (use `none` to remove a skin). The NPC names are:
+    * On all versions except DCv1 and early prototypes: `ninja`, `rico`, `sonic`, `knuckles`, `tails`
+    * On GC, Xbox, and BB: `flowen`, `elly`
+    * On BB only: `momoka`, `irene`, `guild`, `nurse`
+* `$edit secid SECID-NAME`: Set your section ID (cheat mode is required for this unless your character is Level 1)
+
+The remaining subcommands are only available if cheat mode is enabled on the server. They are:
+* `$edit atp N`: Set your ATP to N until stats are updated (e.g. by leveling up)
+* `$edit mst N`: Set your MST to N until stats are updated
+* `$edit evp N`: Set your EVP to N until stats are updated
+* `$edit dfp N`: Set your DFP to N until stats are updated
+* `$edit ata N`: Set your ATA to N until stats are updated
+* `$edit lck N`: Set your LCK to N until stats are updated
+* `$edit hp N`: Set your MST to N until stats are updated
+* `$edit meseta N`: Set the amount of Meseta in your inventory
+* `$edit exp N`: Set your total amount of EXP (does not affect level)
+* `$edit level N`: Set your current level (recomputes stats, but does not affect EXP)
+* `$edit tech TECH-NAME LEVEL`: Set the level of one of your techniques
 
 # Non-server features
 
@@ -463,6 +680,7 @@ The data formats that newserv can convert to/from are:
 | PSO GC quest file (.gci)       | None                      | `decode-gci`                 |
 | Download quest file (.dlq)     | None                      | `decode-dlq`                 |
 | Server quest file (.qst)       | `encode-qst`              | `decode-qst`                 |
+| PSO DC save file (.vms)        | `encrypt-vms-save`        | `decrypt-vms-save`           |
 | PSO PC save file               | `encrypt-pc-save`         | `decrypt-pc-save`            |
 | PSO GC save file (.gci)        | `encrypt-gci-save`        | `decrypt-gci-save`           |
 | PSO GC snapshot file           | None                      | `decode-gci-snapshot`        |
@@ -489,3 +707,108 @@ There are several actions that don't fit well into the table above, which let yo
 * Convert item data to a human-readable description, or vice versa (`describe-item`)
 * Connect to another PSO server and pretend to be a client (`cat-client`)
 * Generate or describe DC serial numbers (`generate-dc-serial-number`, `inspect-dc-serial-number`)
+
+# Docker
+Docker is new and mostly unsupported at this time. However, here are some best-effort steps to build and run in a docker container on Ubuntu Linux.
+Tested on Ubuntu 22.04.4 LTS.
+Note: You cannot have anything except this docker container using port 53 (DNS) on your server.
+
+Install prerequisites
+```
+sudo apt install -y git
+sudo apt install -y cmake. ## minimum version is 3.10. Check installed version with "cmake --version"
+```
+
+Clone repository
+```
+cd ~
+git clone https://github.com/fuzziqersoftware/newserv/
+cd ~/newserv
+```
+
+Build newserv. This will take a while. Don't forget the period at the end!
+```
+sudo docker build -t newserv .
+```
+
+Create persistent directories. Assuming you want to store the persistent data in your home directory
+```
+mkdir ~/newservPersist
+mkdir ~/newservPersist/players
+mkdir ~/newservPersist/teams
+mkdir ~/newservPersist/licenses
+```
+
+Copy config file to config dir
+```
+cp ~/newserv/system/config.example.json ~/newservPersist/config.json
+```
+
+Edit config.json
+```
+nano ~/newservPersist/config.json
+```
+Pro tip:
+Set "LocalAddress" to the static, LAN IP address of your server. If your server LAN IP is "192.168.0.10":
+"LocalAddress": "192.168.0.10",
+
+Set "ExternalAddress" to the WAN IP address of your network. If your WAN IP is "8.8.8.8":
+"ExternalAddress": "8.8.8.8",
+
+For Dolphin > Settings. Set SP1 to "Broadband Adapter (HLE)" Click [...] next to this, and set the DNS to the IP address of your server. Then start the game. Changes will not take affect if the game is running.
+
+Docker run. Remember to change /home/changeme/newservPersist to your persistent directory. Do not use aliases such as '~'
+```
+docker run --name newserv -p 53:53/udp -p 5100:5100 -p 5110:5110 -p 5111:5111 -p 5112:5112 -p 9064:9064 -p 9100:9100 -p 9103:9103 -p 9300:9300 -p 11000:11000 -p 12000:12000 -p 12004:12004 -p 12005:12005 -v /etc/localtime:/etc/localtime:ro -v /home/changeme/newservPersist/config.json:/newserv/system/config.json -v /home/changeme/newservPersist/players:/newserv/system/players -v /home/changeme/newservPersist/teams:/newserv/system/teams -v /home/changeme/newservPersist/licenses:/newserv/system/licenses --restart no newserv:latest
+```
+
+Docker run host network mode. Remember to change /home/changeme/newservPersist to your persistent directory. Do not use aliases such as '~'
+```
+docker run --net host --name newserv -v /etc/localtime:/etc/localtime:ro -v /home/changeme/newservPersist/config.json:/newserv/system/config.json -v /home/changeme/newservPersist/players:/newserv/system/players -v /home/changeme/newservPersist/teams:/newserv/system/teams -v /home/changeme/newservPersist/licenses:/newserv/system/licenses --restart no newserv:latest
+```
+
+Docker compose. Remember to change /home/changeme/newservPersist to your persistent directory. Do not use aliases such as '~'
+```
+name: psonewserv
+services:
+    newserv:
+        container_name: newserv
+        ports:
+            - 53:53/udp
+            - 5100:5100
+            - 5110:5110
+            - 5111:5111
+            - 5112:5112
+            - 9064:9064
+            - 9100:9100
+            - 9103:9103
+            - 9300:9300
+            - 11000:11000
+            - 12000:12000
+            - 12004:12004
+            - 12005:12005
+        volumes:
+            - /etc/localtime:/etc/localtime:ro
+            - /home/changeme/newservPersist/config.json:/newserv/system/config.json
+            - /home/changeme/newservPersist/players:/newserv/system/players
+            - /home/changeme/newservPersist/teams:/newserv/system/teams
+            - /home/changeme/newservPersist/licenses:/newserv/system/licenses
+        restart: no ## Set to whatever you want.
+        image: newserv:latest
+```
+Docker compose host network mode. Remember to change /home/changeme/newservPersist to your persistent directory. Do not use aliases such as '~'
+```
+name: psonewserv
+services:
+    newserv:
+        container_name: newserv
+        volumes:
+            - /etc/localtime:/etc/localtime:ro
+            - /home/changeme/newservPersist/config.json:/newserv/system/config.json
+            - /home/changeme/newservPersist/players:/newserv/system/players
+            - /home/changeme/newservPersist/teams:/newserv/system/teams
+            - /home/changeme/newservPersist/licenses:/newserv/system/licenses
+        restart: no ## Set to whatever you want.
+        network_mode: host
+        image: newserv:latest
+```

TODO.md

@@ -1,10 +1,9 @@
 ## General
 
-- Make reloading happen on separate threads so compression doesn't block active clients
-- Implement decrypt/encrypt actions for VMS files
 - Make UI strings localizable (e.g. entries in menus, welcome message, etc.)
 - Add an idle connection timeout for proxy sessions
-- Clean up ItemParameterTable implementation (see comment ad the top of the class definition)
+- Clean up ItemParameterTable implementation (see comment at the top of the class definition)
+- Handle MeetUserExtensions properly in 41 and C4 commands on the proxy (rewrite the embedded 19 command and store a map of received destinations)
 
 ## PSO DC
 
@@ -13,16 +12,18 @@
 ## Episode 3
 
 - Enforce tournament deck restrictions (e.g. rank checks, No Assist option) when populating COMs at tournament start time
-- Make `reload licenses` not vulnerable to online players' licenses overwriting licenses on disk somehow
+- Make `reload accounts` not vulnerable to online players' accounts overwriting accounts on disk somehow
 - Implement ranks (based on total Meseta earned)
-- Support Trial Edition battles
+- Make an AR code that gets rid of the SAMPLE overlays on NTE
 
 ## PSO XBOX
 
 - Fix receiving Guild Cards from non-Xbox players
 - Research the F94D quest opcode
+- Finish porting the remaining GC patches
 
 ## PSOBB
 
-- Test all quest item subcommands
 - Figure out why Pouilly Slime EXP doesn't work
+- Make server-specified rare enemies work with maps loaded by the proxy
+- Implement serialization for various table types (ItemPMT, ItemPT, etc.)

notes/AITalk.bin-format.txt

@@ -1,16 +0,0 @@
-struct AITalkBin {
-  be_uint32_t num_scs;
-  be_uint32_t sc_offsets[num_scs];
-
-  struct SCDialogueEntry {
-    be_uint32_t num_entries;
-    be_uint32_t unknown_a1;
-    be_uint32_t size; // in bytes
-    struct WhenEntry {
-      be_uint32_t when;
-      be_uint32_t percent_chance; // 0-100
-      be_uint32_t count;
-      be_uint32_t string_ids[count];
-    } __attribute__((packed));
-  } __attribute__((packed));
-} __attribute__((packed));

notes/ar-codes.txt

@@ -1,29 +1,107 @@
-(Ep1&2 USA) Unlock all songs in BGM test
+This file contains client patches I've made for various versions of PSO.
+
+All BB patches are for the JP 1.25.13 version (Tethealla client).
+See also https://github.com/Solybum/Blue-Burst-Patch-Project
+
+(DCv2-US) Disable serial number validation (untested)
+8C1E743E 01E0
+8C2670B6 01E0
+
+(BB) Disable item equip restrictions ("God of equip")
+Memory: 005C9F31 E9A7000000
+File:   001C9331 E9A7000000
+
+All rareable enemies are rare (GC US v1.1)
+040AC944 60000000 // Hildeblue
+040C1B70 60000000 // Rappies
+040C3FC8 60000000 // Nar Lily
+040EB050 48000010 // Pouilly Slime
+
+Unlock all songs in BGM test
 (Note: sadly, there are no secret/unused ones)
-04368960 38600001
-04368964 4E800020
+Ep12-JP12 => 04367A68 38600001
+             04367A6C 4E800020
+Ep12-JP13 => 04368ED8 38600001
+             04368EDC 4E800020
+Ep12-JP14 => 0436A434 38600001
+             0436A438 4E800020
+Ep12-JP15 => 0436A1E8 38600001
+             0436A1EC 4E800020
+Ep12-US10 => 0436891C 38600001
+             04368920 4E800020
+Ep12-US11 => 04368960 38600001
+             04368964 4E800020
+Ep12-US12 => 0436A5B4 38600001
+             0436A5B8 4E800020
+Ep12-EU   => 043699A8 38600001
+             043699AC 4E800020
+Ep3-NTE   => 041EA948 38600001
+             041EA94C 4E800020
+Ep3-JP    => 041D8CF0 38600001
+             041D8CF4 4E800020
+Ep3-US    => 041D8D7C 38600001
+             041D8D80 4E800020
+Ep3-EU    => 041D93F0 38600001
+             041D93F4 4E800020
 
-(Ep1&2 USA v1.1) Play lobby (and event) music on Pioneer 2 also
-0417E0F0 60000000
+Play lobby (and event) music in Morgue also
+Ep12-JP12 => 0417DD34 60000000
+Ep12-JP13 => 0417E0E8 60000000
+Ep12-JP14 => 0417E24C 60000000
+Ep12-JP15 => 0417E1AC 60000000
+Ep12-US10 => 0417E0F0 60000000
+Ep12-US11 => 0417E0F0 60000000
+Ep12-US12 => 0417E210 60000000
+Ep12-EU   => 0417E6D4 60000000
+Ep3-NTE   => 040B8C7C 60000000
+Ep3-US    => 040B7028 60000000
+Ep3-JP    => 040B7044 60000000
+Ep3-EU    => 040B746C 60000000
 
-(Ep3 USA) Play lobby (and event) music in Morgue also
-040B7028 60000000
+Skip white logo screens during startup
+Ep12-JP12 => 0413EE54 38000007
+Ep12-JP13 => 0413F1DC 38000007
+Ep12-JP14 => 0413F338 38000007
+Ep12-JP15 => 0413F298 38000007
+Ep12-US10 => 0413F190 38000007
+Ep12-US11 => 0413F190 38000007
+Ep12-US12 => 0413F2A8 38000007
+Ep12-EU   => 0413F524 38000007
+Ep3-NTE   => 0409E10C 38000007
+Ep3-JP    => 0409D810 38000007
+Ep3-US    => 0409D774 38000007
+Ep3-EU    => 0409D9A4 38000007
 
-(Ep3 USA) Skip white logo screens during startup
-0409D774 38000007
-(Episodes 1&2 USA v1.1) Skip white logo screens during startup
-0413F190 38000007
+Skip agreement prompts before online game
+Ep12-JP12 => 0432737C 38000003
+Ep12-JP13 => 043283CC 38000003
+Ep12-JP14 => 043298E8 38000003
+Ep12-JP15 => 04329690 38000003
+Ep12-US10 => 04327D3C 38000003
+Ep12-US11 => 04327D80 38000003
+Ep12-US12 => 0432984C 38000003
+Ep12-EU   => 04328C58 38000003
+Ep3-NTE   => 041C67D0 38000003
+Ep3-JP    => 041B5234 38000003
+Ep3-US    => 041B50C8 38000003
+Ep3-EU    => 041B574C 38000003
 
-(Ep3 USA) Skip agreement prompts before online game
-041B50C8 38000003
-(Episodes 1&2 USA v1.1) Skip agreement prompt before online game
-04327D80 38000003
+Disable rate limit for pressing A during loading screens
+Ep3-NTE => 042E1030 38000000
+Ep3-JP  => 042F8BE4 38000000
+Ep3-US  => 042F9B30 38000000
+Ep3-EU  => 042FA734 38000000
 
-(Ep3 USA) Disable rate limit for pressing A during loading screens
-042F9B30 38000000
+Auto-press A as fast as possible during loading screens
+Ep3-EU  => 042FA6C4 60000000
+Ep3-US  => 042F9AC0 60000000
+Ep3-NTE => 040C2C48 60000000
+Ep3-JP  => 042F8B74 60000000
 
-(Ep3 USA) Auto-press A as fast as possible during loading screens
-042F9AC0 60000000
+(Ep1&2 USA v1.1) Change type of all loading screens
+0401CA04 3BE0000X
+0401CA08 48000038
+Values for X: 0 = lobby/game join, 1 = quest load, 3 = pipe up, 4 = pipe down, anything else = silent black screen
 
 (Ep3 USA) Replace loading screen A button sounds with random sounds
 042F9B18 4804BB19
@@ -32,6 +110,13 @@
 042F9B24 64630005
 042F9B28 38800000
 
+(Ep3 NTE) Replace loading screen A button sounds with random sounds
+042E1018 480309A9
+042E101C 5463063E
+042E1020 60631400
+042E1024 64630005
+042E1028 38800000
+
 (Ep3 USA) Change color of loading screens
 (Replace AA, RR, GG, BB appropriately)
 042FA704 3CC0AARR
@@ -43,11 +128,16 @@
 0400BD64 EC5D00B2
 0400BD68 4E800020
 
-(Ep3 USA) Disable darkening effect during battle details mode
-042F951C 4E800020
+Disable darkening effect during battle details mode
+Ep3-NTE => 042E09D8 4E800020
+Ep3-JP  => 042F85D0 4E800020
+Ep3-US  => 042F951C 4E800020
+Ep3-EU  => 042FA120 4E800020
 
-(Ep3 USA) Unlock all COM decks
-042CA908 38600001
+Unlock all COM decks
+Ep3-JP => 042C9B34 38600001
+Ep3-EU => 042CB414 38600001
+Ep3-US => 042CA908 38600001
 
 (Ep3 USA) Enable all lobby counter options in non-CARD lobbies
 04096A8C 480000C0
@@ -103,8 +193,11 @@
 040002BC 7C633050
 040002C0 4E800020
 
-(Ep3 USA) Unlock all offline free battle maps
-042CAA00 38600001
+Unlock all offline free battle maps
+Ep3-NTE => 042BE538 38600001
+Ep3-JP =>  042C9C2C 38600001
+Ep3-EU =>  042CB50C 38600001
+Ep3-US =>  042CAA00 38600001
 (This unlocks ALL maps, including a bunch of maps with garbage names that crash if you try to play them)
 
 (Ep3 USA) Talk to auction counter offline to get all cards
@@ -165,8 +258,10 @@
 025CB6AA 00000001
 TODO: Figure out more debug message conditionals (vars/functions) and add them here
 
-(Episode 3 USA) Able to find VIP cards offline (but they're still rare)
-042C0B20 4800000C
+Able to find VIP cards offline (but they're still rare)
+Ep3-EU => 042C15DC 4800000C
+Ep3-JP => 042BFE24 4800000C
+Ep3-US => 042C0B20 4800000C
 
 (Ep3 USA) Hold L when starting battle to enter debug menu
 042C5460 4BD3AF78
@@ -181,10 +276,14 @@ TODO: Figure out more debug message conditionals (vars/functions) and add them h
 040003F8 3800001A
 040003FC 482C5068
 
-(Ep3 USA) Dressing room always accessible
-041A16FC 38600001
+Dressing room always accessible
+Ep3-NTE => 041B2A2C 38600001
+Ep3-JP  => 041A1920 38600001
+Ep3-EU  => 041A1C84 38600001
+Ep3-US  => 041A16FC 38600001
 
 (Ep3 USA) Full dressing room v1
+Original Ep1&2 code by Ralf @ GC-Forever
 Can't change your class, but you start with your existing appearance
 Go online with this code on after using the dressing room to fully save changes
 0418EB5C 60000000
@@ -192,6 +291,7 @@ Go online with this code on after using the dressing room to fully save changes
 042A0188 387E2120
 
 (Ep3 USA) Full dressing room v2
+Original Ep1&2 code by Ralf @ GC-Forever
 Can change your class, but you start with the default appearance
 Go online with this code on after using the dressing room to fully save changes
 04186ECC 4BFFFFD8
@@ -201,8 +301,11 @@ Go online with this code on after using the dressing room to fully save changes
 (Ep3 USA) Replace Options menu with debug menu
 04149E70 38600019
 
-(Ep3 USA) Jukebox is free
-0430D1DC 48000024
+Jukebox is free
+Ep3-NTE => 042248C4 48000024 (useless because the jukebox isn't loaded in NTE, but apparently the code for it exists)
+Ep3-JP  => 0430C178 48000024
+Ep3-US  => 0430D1DC 48000024
+Ep3-EU  => 0430DE3C 48000024
 
 (Ep3 USA) Use own character in battle (online only)
 041FFAB0 4800001C
@@ -233,8 +336,11 @@ Go online with this code on after using the dressing room to fully save changes
 0412F8D4 7D0803A6
 0412F8D8 4BEDEBF4
 
-(Ep3 USA) Metal tiles don't appear in Simulator map
-04296904 4E800020
+Metal tiles don't appear in Simulator map
+Ep3-NTE => 0428FED8 4E800020
+Ep3-JP  => 04296054 4E800020
+Ep3-US  => 04296904 4E800020
+Ep3-EU  => 04297278 4E800020
 
 (Ep3 USA) Enable Boooo and Laughter soundchat sounds
 Note: Without a TextEnglish.pr2/pr3 patch, the menu items for these sounds will be blank (but they will still work)
@@ -270,3 +376,8 @@ Note: Without a TextEnglish.pr2/pr3 patch, the menu items for these sounds will
 0408E448 38000001
 0408E44C 900DA62C
 0408E450 4E800020
+
+(v1.1 USA) Replace all sound effects with specified sound effect
+042256E4 3F40XXXX
+042256E8 635AYYYY
+042256EC 4800000C

notes/bb-banned-items.txt

@@ -0,0 +1,80 @@
+000F04 LOGiN
+006E00 GAME MAGAZNE
+00AD00 RAGE DE FEU
+00AD01 RAGE DE FEU
+00AD02 RAGE DE FEU
+00D000 UNKNOWN3
+00D100 UNKNOWN4
+01013D KROE'S SWEATER
+01013F SONICTEAM ARMOR
+010230 HUNTER'S SHELL
+010233 HUNTER'S SHELL
+010234 HUNTER'S SHELL
+010236 Barrier
+010237 Barrier
+010238 Barrier
+010239 Barrier
+010253 BLUE RING
+010254 BLUE RING
+010255 BLUE RING
+010256 BLUE RING
+010257 BLUE RING
+010258 BLUE RING
+01025A BLUE RING
+01025B GREEN RING
+01025C GREEN RING
+01025D GREEN RING
+01025E GREEN RING
+010260 GREEN RING
+010261 GREEN RING
+010262 GREEN RING
+010263 YELLOW RING
+010264 YELLOW RING
+010265 YELLOW RING
+010267 YELLOW RING
+010268 YELLOW RING
+010269 YELLOW RING
+01026A YELLOW RING
+01026B PURPLE RING
+01026D PURPLE RING
+01026E PURPLE RING
+01026F PURPLE RING
+010270 PURPLE RING
+010271 PURPLE RING
+010272 PURPLE RING
+010274 WHITE RING
+010276 WHITE RING
+010277 WHITE RING
+010278 WHITE RING
+010279 WHITE RING
+01027A WHITE RING
+01027C BLACK RING
+01027D BLACK RING
+01027E BLACK RING
+01027F BLACK RING
+010281 BLACK RING
+01029A UNKNOWN_B
+024300 \n
+024A00 Yahoo!
+024D00 Cell of MAG 0503
+024E00 Cell of MAG 0504
+024F00 Cell of MAG 0505
+025000 Cell of MAG 0506
+025100 Cell of MAG 0507
+03120B New Year's Card
+03120C Christmas Card
+03120D Birthday Card
+03120E Proof of Sonic Team
+03120F Special Event Ticket
+03140A Bouquet
+03140B Decoction
+031603 DISK Vol.4 "Open Your Heart"
+031604 DISK Vol.5 "Live & Learn"
+031801 UNKNOWN2
+031808 Yahoo!'s engine
+03180B Cell of MAG 0503
+03180C Cell of MAG 0504
+03180D Cell of MAG 0505
+03180E Cell of MAG 0506
+03180F Cell of MAG 0507
+200000 (invalid item code)

notes/ep3-entry-counter-flags.txt

@@ -0,0 +1,9 @@
+entry counter flags
+
+01 = rules have any non-default values
+02 = map number is set
+04 = UNKNOWN (something to do with deck selection/verification)
+08 = tournament mode (set by 6xB4x3D; shows timer in battle select menu and skips map select and rule select)
+10 = UNKNOWN (used by 6xB5x43)
+20 = command DC received
+40 = tournament result available (6xB4x51 received)

notes/gc-nte-areas.txt

@@ -0,0 +1,18 @@
+    Ep1                Ep2
+1   Forest 1           Temple
+2   Forest 2           Temple
+3   Cave 1             Spaceship
+4   Cave 2             Spaceship
+5   Cave 3             CCA
+6   Mine 1             Jungle
+7   Mine 2             Jungle
+8   Ruins 1 (broken)   Mountain
+9   Ruins 2 (broken)   Seaside
+10  Ruins 3 (broken)   Void (Seabed doors + Mine music)
+11  Dragon             Void (doors + Dolmolm + Mine music)
+12  De Rol Le          Gal Gryphon
+13  Vol Opt            Olga Flow (unfinished, Flow does no damage)
+14  void (Falz music)  Barba Ray (unfinished)
+15  Lobby              Gol Dragon (unfinished)
+16  Versus1            crash
+17  Versus2            crash

notes/hangame-notes.txt

@@ -0,0 +1,19 @@
+patch required in TethVer12513 to get this to work: 0048210D EB
+
+is_hangame callsites:
+0040457C - ??? (something in TDataProtocol?)
+004820F4 - client version check (use patch above to bypass)
+00708318 - patch server domain name
+00708348 - patch server port
+0070852C - ep4 unlocked setting (always true for hangame)
+007085F4 - data server domain name
+00708670 - data server port
+007618E3 - whether to save user/pass to registry
+00761C4C - create title screen menu (only shows Start Game and Exit Game in Hangame mode)
+007623B0 - input password length limit?? (does nothing, since both branches of if statement lead to same result)
+00762530 - registry account data access
+00762708 - input password length limit?? (does nothing, since both branches of if statement lead to same result)
+0076296F - input username length limit?? (limits to 12 instead of 16)
+00762C30 - input username length limit?? (limits to 12 instead of 16)
+00762D00 - password length limit again??
+00762D2C - username length limit again??

notes/patch-ports.txt

@@ -73,9 +73,9 @@ ItemLossPrevention
 *** desc=Don't lose items if\nyou don't log off\nnormally
 JP12------------- JP13------------- JP14------------- JP15------------- US10------------- US11------------- US12------------- EU--------------- DISASSEMBLY (US10)
 801D33E4 4800004C 801D38EC 4800004C 801D3CC4 4800004C 801D39B8 4800004C 801D381C 4800004C 801D381C 4800004C 801D3A1C 4800004C 801D3ED8 4800004C b         +0x0000004C /* 801D3868 */
-                                    8020010C 60000000 801FF710 60000000 801FF0FC 60000000 801FF0FC 60000000 801FFA44 60000000 801FF9E0 60000000 nop
-                                    802016CC 60000000 80200C9C 60000000 80200658 60000000 80200658 60000000 80200FD0 60000000 80200F3C 60000000 nop
-                                    801FD944 38000000 80202860 38000000 802021C4 38000000 802021C4 38000000 80202B94 38000000 80202AA8 38000000 li        r0, 0x0000
+801FE900 60000000 801FF174 60000000 8020010C 60000000 801FF710 60000000 801FF0FC 60000000 801FF0FC 60000000 801FFA44 60000000 801FF9E0 60000000 nop
+801FFE5C 60000000 802006D0 60000000 802016CC 60000000 80200C9C 60000000 80200658 60000000 80200658 60000000 80200FD0 60000000 80200F3C 60000000 nop
+802019C8 38000000 8020223C 38000000 801FD944 38000000 80202860 38000000 802021C4 38000000 802021C4 38000000 80202B94 38000000 80202AA8 38000000 li        r0, 0x0000
 802C2060 4800004C 802C2F98 4800004C 802C42E4 4800004C 802C3E78 4800004C 802C2A40 4800004C 802C2A84 4800004C 802C402C 4800004C 802C37C0 4800004C b         +0x0000004C /* 802C2A8C */
 802D0AA0 48000020 802D1A58 48000020 802D2C10 48000020 802D2938 48000020 802D1480 48000020 802D14C4 48000020 802D2AEC 48000020 802D2280 48000020 b         +0x00000020 /* 802D14A0 */
 
@@ -754,15 +754,45 @@ Show Enemy HP Bars
 EnemyHPBars
 *** name=Enemy HP bars
 *** desc=Show HP bars in\nenemy info windows
-JP12------------- JP13------------- JP14------------- JP15------------- US10------------- US11------------- US12------------- EU--------------- DISASSEMBLY (US10)
+JP12------------- JP13------------- JP14------------- JP15------------- US10------------- US11------------- US12------------- EU--------------- DISASSEMBLY (US12)
 802612C4 4BFE1541 80261E9C 4BFE1349 80262EE4 4BFE0665 80262C98 4BFE1241 80261B9C 4BFE1545 80261B9C 4BFE1545 80262F5C 4BFE12B1 802627A4 4BFE12B1 bl        -0x0001EABC /* 802430E0 */
-804CAF00 42300000 804CE650 42300000 804D0BA0 42300000 804D0940 42300000 804CB6D0 42300000 804CBBB0 42300000 804D0218 42300000 804D0608 42300000 bdnz      cr4, +0x00000000 /* 804CB6D0 */
-804CAF1C FF00FF15 804CE66C FF00FF15 804D0BBC FF00FF15 804D095C FF00FF15 804CB6EC FF00FF15 804CBBCC FF00FF15 804D0234 FF00FF15 804D0624 FF00FF15 .invalid  FC, 0
-805CBFBC 42A00000 805D65BC 42A00000 805DDA5C 42A00000 805DD7FC 42A00000 805CC8C4 42A00000 805D38E4 42A00000 805DD104 42A00000 805D9344 42A00000 b         +0x00000000 /* 805CC8C4 */
-804CAE40 42640000 804CE590 42640000 804D0AE0 42640000 804D0880 42640000 804CB610 42640000 804CBAF0 42640000 804D0158 42640000 804D0548 42640000 bc        19, 4, +0x00000000 /* 804CB610 */
-804CAE4C 42640000 804CE59C 42640000 804D0AEC 42640000 804D088C 42640000 804CB61C 42640000 804CBAFC 42640000 804D0164 42640000 804D0554 42640000 bc        19, 4, +0x00000000 /* 804CB61C */
-804CAE58 42640000 804CE5A8 42640000 804D0AF8 42640000 804D0898 42640000 804CB628 42640000 804CBB08 42640000 804D0170 42640000 804D0560 42640000 bc        19, 4, +0x00000000 /* 804CB628 */
-804CAE64 42640000 804CE5B4 42640000 804D0B04 42640000 804D08A4 42640000 804CB634 42640000 804CBB14 42640000 804D017C 42640000 804D056C 42640000 bc        19, 4, +0x00000000 /* 804CB634 */
+804CAF00 42780000 804CE650 42780000 804D0BA0 42780000 804D0940 42780000 804CB6D0 42780000 804CBBB0 42780000 804D0218 42780000 804D0608 42780000
+804CAF1C FF00FF15 804CE66C FF00FF15 804D0BBC FF00FF15 804D095C FF00FF15 804CB6EC FF00FF15 804CBBCC FF00FF15 804D0234 FF00FF15 804D0624 FF00FF15
+805CBFBC 42C00000 805D65BC 42C00000 805DDA5C 42C00000 805DD7FC 42C00000 805CC8C4 42C00000 805D38E4 42C00000 805DD104 42C00000 805D9344 42C00000
+804CAE40 42960000 804CE590 42960000 804D0AE0 42960000 804D0880 42960000 804CB610 42960000 804CBAF0 42960000 804D0158 42960000 804D0548 42960000
+804CAE4C 42960000 804CE59C 42960000 804D0AEC 42960000 804D088C 42960000 804CB61C 42960000 804CBAFC 42960000 804D0164 42960000 804D0554 42960000
+804CAE58 42960000 804CE5A8 42960000 804D0AF8 42960000 804D0898 42960000 804CB628 42960000 804CBB08 42960000 804D0170 42960000 804D0560 42960000
+804CAE64 42960000 804CE5B4 42960000 804D0B04 42960000 804D08A4 42960000 804CB634 42960000 804CBB14 42960000 804D017C 42960000 804D056C 42960000
+804CAE70 42960000 804CE5C0 42960000 804D0B10 42960000 804D08B0 42960000 804CB640 42960000 804CBB20 42960000 804D0188 42960000 804D0578 42960000
+80261260 4BDAA3F1 80261E38 4BDA9819 80262E80 4BDA87D1 80262C34 4BDA8A1D 80261B38 4BDA9B19 80261B38 4BDA9B19 80262EF8 4BDA8759 80262740 4BDA8F11 bl        -0x002578A8 /* 8000B650 */
+80261420 4BDAA245 80261FF8 4BDA966D 80263040 4BDA8625 80262DF4 4BDA8871 80261CF8 4BDA996D 80261CF8 4BDA996D 802630B8 4BDA85AD 80262900 4BDA8D65 bl        -0x00257A54 /* 8000B664 */
+8000B650 3CA08001 8000B650 3CA08001 8000B650 3CA08001 8000B650 3CA08001 8000B650 3CA08001 8000B650 3CA08001 8000B650 3CA08001 8000B650 3CA08001 lis       r5, 0x8001
+8000B654 8065B6BC 8000B654 8065B6BC 8000B654 8065B6BC 8000B654 8065B6BC 8000B654 8065B6BC 8000B654 8065B6BC 8000B654 8065B6BC 8000B654 8065B6BC lwz       r3, [r5 - 0x4944]
+8000B658 7FFEFB78 8000B658 7FFEFB78 8000B658 7FFEFB78 8000B658 7FFEFB78 8000B658 7FFEFB78 8000B658 7FFEFB78 8000B658 7FFEFB78 8000B658 7FFEFB78 mr        r30, r31
+8000B65C A8DE032C 8000B65C A8DE032C 8000B65C A8DE032C 8000B65C A8DE032C 8000B65C A8DE032C 8000B65C A8DE032C 8000B65C A8DE032C 8000B65C A8DE032C lha       r6, [r30 + 0x032C]
+8000B660 48000010 8000B660 48000010 8000B660 48000010 8000B660 48000010 8000B660 48000010 8000B660 48000010 8000B660 48000010 8000B660 48000010 b         +0x00000010 /* 8000B670 */
+8000B664 A8DE02B8 8000B664 A8DE02B8 8000B664 A8DE02B8 8000B664 A8DE02B8 8000B664 A8DE02B8 8000B664 A8DE02B8 8000B664 A8DE02B8 8000B664 A8DE02B8 lha       r6, [r30 + 0x02B8]
+8000B668 3CA08001 8000B668 3CA08001 8000B668 3CA08001 8000B668 3CA08001 8000B668 3CA08001 8000B668 3CA08001 8000B668 3CA08001 8000B668 3CA08001 lis       r5, 0x8001
+8000B66C 9065B6BC 8000B66C 9065B6BC 8000B66C 9065B6BC 8000B66C 9065B6BC 8000B66C 9065B6BC 8000B66C 9065B6BC 8000B66C 9065B6BC 8000B66C 9065B6BC stw       [r5 - 0x4944], r3
+8000B670 7C0802A6 8000B670 7C0802A6 8000B670 7C0802A6 8000B670 7C0802A6 8000B670 7C0802A6 8000B670 7C0802A6 8000B670 7C0802A6 8000B670 7C0802A6 mflr      r0
+8000B674 9005B6C0 8000B674 9005B6C0 8000B674 9005B6C0 8000B674 9005B6C0 8000B674 9005B6C0 8000B674 9005B6C0 8000B674 9005B6C0 8000B674 9005B6C0 stw       [r5 - 0x4940], r0
+8000B678 7C651B78 8000B678 7C651B78 8000B678 7C651B78 8000B678 7C651B78 8000B678 7C651B78 8000B678 7C651B78 8000B678 7C651B78 8000B678 7C651B78 mr        r5, r3
+8000B67C A8FE02B8 8000B67C A8FE02B8 8000B67C A8FE02B8 8000B67C A8FE02B8 8000B67C A8FE02B8 8000B67C A8FE02B8 8000B67C A8FE02B8 8000B67C A8FE02B8 lha       r7, [r30 + 0x02B8]
+8000B680 3C808000 8000B680 3C808000 8000B680 3C808000 8000B680 3C808000 8000B680 3C808000 8000B680 3C808000 8000B680 3C808000 8000B680 3C808000 lis       r4, 0x8000
+8000B684 6084B6AC 8000B684 6084B6AC 8000B684 6084B6AC 8000B684 6084B6AC 8000B684 6084B6AC 8000B684 6084B6AC 8000B684 6084B6AC 8000B684 6084B6AC ori       r4, r4, 0xB6AC
+8000B688 38640018 8000B688 38640018 8000B688 38640018 8000B688 38640018 8000B688 38640018 8000B688 38640018 8000B688 38640018 8000B688 38640018 addi      r3, r4, 0x0018
+8000B68C 4CC63182 8000B68C 4CC63182 8000B68C 4CC63182 8000B68C 4CC63182 8000B68C 4CC63182 8000B68C 4CC63182 8000B68C 4CC63182 8000B68C 4CC63182 crxor     crb6, crb6, crb6
+8000B690 4838A86D 8000B690 4838D275 8000B690 4838F115 8000B690 4838EEC5 8000B690 4838BB3D 8000B690 4838BB95 8000B690 4838F295 8000B690 4838DD85 bl        sprintf /* 8039A924 */
+8000B694 3C808000 8000B694 3C808000 8000B694 3C808000 8000B694 3C808000 8000B694 3C808000 8000B694 3C808000 8000B694 3C808000 8000B694 3C808000 lis       r4, 0x8000
+8000B698 6084B6C4 8000B698 6084B6C4 8000B698 6084B6C4 8000B698 6084B6C4 8000B698 6084B6C4 8000B698 6084B6C4 8000B698 6084B6C4 8000B698 6084B6C4 ori       r4, r4, 0xB6C4
+8000B69C 7F83E378 8000B69C 7F83E378 8000B69C 7F83E378 8000B69C 7F83E378 8000B69C 7F83E378 8000B69C 7F83E378 8000B69C 7F83E378 8000B69C 7F83E378 mr        r3, r28
+8000B6A0 8004FFFC 8000B6A0 8004FFFC 8000B6A0 8004FFFC 8000B6A0 8004FFFC 8000B6A0 8004FFFC 8000B6A0 8004FFFC 8000B6A0 8004FFFC 8000B6A0 8004FFFC lwz       r0, [r4 - 0x0004]
+8000B6A4 7C0803A6 8000B6A4 7C0803A6 8000B6A4 7C0803A6 8000B6A4 7C0803A6 8000B6A4 7C0803A6 8000B6A4 7C0803A6 8000B6A4 7C0803A6 8000B6A4 7C0803A6 mtlr      r0
+8000B6A8 4E800020 8000B6A8 4E800020 8000B6A8 4E800020 8000B6A8 4E800020 8000B6A8 4E800020 8000B6A8 4E800020 8000B6A8 4E800020 8000B6A8 4E800020 blr
+8000B6AC 25730A0A 8000B6AC 25730A0A 8000B6AC 25730A0A 8000B6AC 25730A0A 8000B6AC 25730A0A 8000B6AC 25730A0A 8000B6AC 25730A0A 8000B6AC 25730A0A .invalid
+8000B6B0 48503A25 8000B6B0 48503A25 8000B6B0 48503A25 8000B6B0 48503A25 8000B6B0 48503A25 8000B6B0 48503A25 8000B6B0 48503A25 8000B6B0 48503A25 bl        +0x00503A24 /* 8050F0D4 */
+8000B6B4 642F2564 8000B6B4 642F2564 8000B6B4 642F2564 8000B6B4 642F2564 8000B6B4 642F2564 8000B6B4 642F2564 8000B6B4 642F2564 8000B6B4 642F2564 oris      r15, r1, 0x2564
+8000B6B8 00000000 8000B6B8 00000000 8000B6B8 00000000 8000B6B8 00000000 8000B6B8 00000000 8000B6B8 00000000 8000B6B8 00000000 8000B6B8 00000000 .invalid
 
 PSO DC Reticle Colours
 DCReticleColors

notes/psobb/translated-coren-map-files/npc-coren-textstrings/tsuchan.txt

@@ -1,96 +1,94 @@
-###########################################################
-
-NPC: Coren Tsu - The Wanderer
-AREAS: Pioneer 2
-
-Translations by: apexseals (discord: apexseals)
-Proofing & Debugging by: nolrinale (github.com/nolrinale)
-
-###########################################################
-
-presentation:
-
-I am Coren Tsu, a wandering merchant,
-you could say.
-
-Please take some time to look at
-the rare and wonderous goods
-I have been collecting.
-
-If you spend a little meseta,
-you could win a wonderful prize.
-
-Well? Wanna try?
-
-
-You may win,
-you may lose.
-
-But if you don't win,
-don't take it out on me.
-
-That's just the way
-gambling is, yes?
-
-Well then, how much
-meseta do you want to pay?
-
-As long as you pay me,
-I'll give you a great service.
-
-
-Huh?
-
-That's too bad...
-
-Well, these kind of things usually
-have a chance to lose money. 
-
-Let's keep this discreet.
-If you feel up to it, talk to me again.
-
-
-It seems you have
-too many items.
-
-First, go and
-organize your items,
-
-Then speak to me again.
-
-What?
-
-You said you'd try,
-then you said no.
-
-People like that
-fail at everything.
-
-
-What the...?
-
-You don't have the 
-meseta to pay me?
-
-I won't work with such
-cold hearted people.
-
-
-Alright, let's do it.
-
-You better pray
-for something good...
-
-
-Look here!
-Take it!
-
-Even if you had bad luck,
-something good will come out of it.
-
-You'll win someday!
-
-In case you want to try again,
-come back to me once more.
-
-
+###########################################################
+
+NPC: Coren Tsu - The Wanderer
+AREAS: Pioneer 2
+
+Translations by: apexseals (discord: apexseals)
+Proofing & Debugging by: nolrinale (github.com/nolrinale)
+
+###########################################################
+
+presentation:
+
+I am Coren Tsu, a wandering merchant,
+you could say.
+
+Please take some time to look at
+the rare and wonderous goods
+I have been collecting.
+
+If you spend a little meseta,
+you could win a wonderful prize.
+
+Well? Wanna try?
+
+
+You may win,
+you may lose.
+
+But if you don't win,
+don't take it out on me.
+
+That's just the way
+gambling is, yes?
+
+Well then, how much
+meseta do you want to pay?
+
+As long as you pay me,
+I'll give you a great service.
+
+
+Huh?
+
+That's too bad...
+
+Well, these kind of things usually
+have a chance to lose money.
+
+Let's keep this discreet.
+If you feel up to it, talk to me again.
+
+
+It seems you have
+too many items.
+
+First, go and
+organize your items,
+
+Then speak to me again.
+
+What?
+
+You said you'd try,
+then you said no.
+
+People like that
+fail at everything.
+
+
+What the...?
+
+You don't have the
+meseta to pay me?
+
+I won't work with such
+cold hearted people.
+
+
+Alright, let's do it.
+
+You better pray
+for something good...
+
+
+Look here!
+Take it!
+
+Even if you had bad luck,
+something good will come out of it.
+
+You'll win someday!
+
+In case you want to try again,
+come back to me once more.

notes/quest-counter-fields.txt

@@ -0,0 +1,165 @@
+$qfread
+
+00 000003FF Garon points
+00 0003FC00 Garon button-mashing game score
+00 03FC0000 Garon timing game score
+00 04000000 Garon Tier 1 (10 cards) of Guild Card counter
+00 08000000 Garon Tier 2 (30 cards) of Guild Card counter
+00 10000000 Garon Tier 3 (50 cards) of Guild Card counter
+00 20000000 Garon Tier 4 (100 cards) of Guild Card counter
+00 C0000000 __UNUSED__
+
+01 00000001 Dream Messenger: NiGHTS
+01 00000002 Pioneer Warehouse: ???
+01 00000004 Garon's Shop: Puyo Pop
+01 00000008 Pioneer Warehouse: Chu Chu Challenge
+01 00000010 Reach for the Dream: Chu Chu Puzzle
+01 00000020 Seat of the Heart: Checkpoint (Normal)
+01 00000040 Seat of the Heart: Checkpoint (Sue path)
+01 00000080 Seat of the Heart: Checkpoint (Elly is mad)
+01 00000100 Seat of the Heart: Quest complete (no Sue path)
+01 00000200 Seat of the Heart: Quest complete (Sue path)
+01 00000400 Seat of the Heart: Got Ragol Ring (Normal)
+01 00000800 Seat of the Heart: Checkpoint (Hard)
+01 00000800 White Day: ???
+01 00001000 Blue Star Memories: Future Forecast
+01 00001000 Seat of the Heart: Checkpoint (Sue path)
+01 00002000 Blue Star Memories: Future Bullet
+01 00002000 Seat of the Heart: Checkpoint (Elly is mad)
+01 00004000 Seat of the Heart: Quest complete (no Sue path)
+01 00008000 Seat of the Heart: Quest complete (Sue path)
+01 00010000 Seat of the Heart: Got Ragol Ring (Hard)
+01 00020000 Seat of the Heart: Checkpoint (Very Hard)
+01 00040000 Seat of the Heart: Checkpoint (Sue path)
+01 00080000 Seat of the Heart: Checkpoint (Elly is mad)
+01 00100000 Seat of the Heart: Quest complete (no Sue path)
+01 00200000 Seat of the Heart: Quest complete (Sue path)
+01 003F8000 Beta Lucky Coins
+01 00400000 Seat of the Heart: Got Ragol Ring (Very Hard)
+01 00800000 Seat of the Heart: Checkpoint (Ultimate)
+01 01000000 Seat of the Heart: Checkpoint (Sue path)
+01 02000000 Seat of the Heart: Checkpoint (Elly is mad)
+01 04000000 Seat of the Heart: Quest complete (no Sue path)
+01 08000000 Seat of the Heart: Quest complete (Sue path)
+01 10000000 Seat of the Heart: Got Ragol Ring (Ultimate)
+01 E0000000 __UNUSED__
+
+02 00000001 Pioneer Halloween: Got Jack-O'-Lantern
+02 00000002 Pioneer Halloween: Got cake
+02 00000020 East Tower
+02 00000020 The East Tower: Paganini side quest
+02 00000040 The West Tower: Paganini side quest
+02 00000040 West Tower
+02 00000080 Labyrinthine Trial: White Ring
+02 00000100 Garon's Treachery: Rakonia Stone
+02 00000200 Garon's Treachery: Fragment of Friendship
+02 00000400 Towards the Future: Purple Ring
+02 00000800 Towards the Future: Flower Bouquet
+02 00002000 Heart Of Poumn
+02 00002000 Rappy's Holiday: Heart of Poumn
+02 003FC000 Rappy's Holiday points
+02 07000000 Respective Tomorrow: WIS
+02 08000000 Respective Tomorrow: S/SS Rank
+02 10000000 Towards the Future: Black Ring
+02 20000000 Green Ring
+02 C0C0101C __UNUSED__
+
+03 000000FF Lucky Tickets
+03 003FFF00 Kill count
+03 07C00000 Song count
+03 08000000 Couple flag
+03 7FFFFFFF MA4 kills (Central Dome)
+03 80000000 __UNUSED__
+
+04 7FFFFFFF MA4 kills (Gal Da Val)
+04 80000000 __UNUSED__
+
+05 00007FFF Principal's Gift: Random Candy ID
+05 00008000 Candy ID init flag
+05 00200000 Racket
+05 00400000 Tree Clippers
+05 00800000 Synthesizer
+05 01000000 Shichishito
+05 02000000 Dirty Life Jacket
+05 04000000 Lost Hell Pallasch: Gush Raygun
+05 F81F0000 __UNUSED__
+
+06 0FF00000 Lucky Tickets
+06 F00FFFFF __UNUSED__
+
+07 00000001 Government 4-5: Normal cleared
+07 00000002 Government 4-5: Hard cleared
+07 00000004 Government 4-5: Very Hard cleared
+07 00000008 Government 4-5: Ultimate cleared
+07 00000010 Government 8-3: Normal cleared
+07 00000020 Government 8-3: Hard cleared
+07 00000040 Government 8-3: Very Hard cleared
+07 00000080 Government 8-3: Ultimate cleared
+07 FFFFFF00 __UNUSED__
+
+08 7FFFFFFF MA4 kills (Crater)
+08 80000000 __UNUSED__
+
+09 00003FFF MA1v2 points
+09 00003FFF Maximum Attack 1 Ver.2: points
+09 0FFFC000 MA2v2 points
+09 0FFFC000 Maximum Attack 2 Ver.2: points
+09 10000000 AOL CUP -Sunset Base- (Mag Cell)
+09 10000000 Maximum Attack 1 Ver.2: Class Master flag
+09 20000000 AOL CUP -Sunset Base- (Ruins)
+09 20000000 Maximum Attack 2 Ver.2: ID Master flag
+09 40000000 Beach Laughter: Got 5 Photon Spheres & Black Ring
+09 40000000 Blue Ring
+09 80000000 __UNUSED__
+
+0A 00000001 Heart of HUmar
+0A 00000002 Heart of HUnewearl
+0A 00000004 Heart of HUcast
+0A 00000008 Heart of HUcaseal
+0A 00000010 Heart of RAmar
+0A 00000020 Heart of RAmarl
+0A 00000040 Heart of RAcast
+0A 00000080 Heart of RAcaseal
+0A 00000100 Heart of FOmar
+0A 00000200 Heart of FOmarl
+0A 00000400 Heart of FOnewm
+0A 00000800 Heart of FOnewearl
+0A 00001000 Heart of Viridia
+0A 00002000 Heart of Greenill
+0A 00004000 Heart of Skyly
+0A 00008000 Heart of Bluefull
+0A 00010000 Heart of Purplenum
+0A 00020000 Heart of Pinkal
+0A 00040000 Heart of Redria
+0A 00080000 Heart of Oran
+0A 00100000 Heart of Yellowboze
+0A 00200000 Heart of Whitill
+0A 7FC00000 Lucky Tickets
+0A 80000000 __UNUSED__
+
+0B 00000001 Garon's Shop: Black Gear
+0B 00000001 Roulette (SEIRYU)
+0B 00000002 Beta -> Final Lucky Coins init flag
+0B 00000002 Roulette (GENBU)
+0B 000001FC Lucky Coins
+0B 0007FC00 Pioneer Christmas ???
+0B 00080000 Cleared 4th Pioneer Christmas tier?
+0B 1FF00000 Wrapping Papers
+0B 20000000 Pioneer Christmas Present
+0B 40000000 Wall
+0B 40000000 White Day: Flower Bouquet or Heart Key
+0B 80000200 __UNUSED__
+
+0C FFFFFFFF __UNUSED__
+
+0D FFFFFFFF __UNUSED__
+
+0E 7FFFFFFF MA4 kills (Total)
+0E 80000000 __UNUSED__
+
+0F 000000FF MA4 Tickets
+0F 00000100 MA4 PHOTON CRYSTAL
+0F 00000200 MA4 FRIEND RING
+0F 00000400 MA4 GIRASOLE
+0F 00000800 MA4 SAMURAI ARMOR
+0F FFFFF000 __UNUSED__

notes/quest-flags.txt

@@ -0,0 +1,188 @@
+0007 = Set by rico capsule in caves
+000B = P2 Tyrell Start
+000C = P2 Irene Start
+000D = P2 Scientist 1 Start
+000E = P2 Scientist 2 Start
+000F = P2 More Scientist stuff.
+0010 = P2 Irene after talking to Tyrell
+0011 = Read a rico capsule (any)
+0012 = P2 Scientist after talking to Irene.
+0013 = P2 Menu 6, quest counter / Tekker talked to
+0014 = Entered Forest 1
+0015 = Entered Forest 2
+0016 = Entered Dragon Area
+0017 = Dragon defeated
+0018 = Caves unlocked
+0018 = P2 Principle after defeating dragon
+0019 = P2 Scientist after defeating dragon
+001E = Entered Caves 1 (Gov 2-1)
+001F = Entered De Rol Le in 2-4
+0020 = De Ro lee defeated
+0021 = Mines unlocked (P2 Tyrell after defeating De Rol Le)
+0028 = Entered Mines 1
+0029 = Entered Vol Opt Area
+002A = Defeated Vol Opt
+002B = Set by rico capsule about the 3 seals (after vol opt).
+002C = Activated Forest monument
+002D = Activated Caves monument (Gov 2-2)
+002E = Activated Mines monument
+002F = Activated all monuments
+0030 = Entered Ruins 1
+0032 = Entered Falz 1
+0035 = Hard mode unlocked
+0036 = Entered Falz 3 // Very Hard mode unlocked (?)
+0037 = Ultimate unlocked
+0046 = One CCA door lock unlocked
+0047 = One CCA door lock unlocked
+0048 = One CCA door lock unlocked
+0049 = Entered Laboratory
+004A = Lab Assistant Start
+004B = Entered Temple Beta
+004C = Defeated Barba Ray
+004D = Lab Assistant after defeating barba ray
+004E = Entered Spaceship Beta
+004F = Defeated Gol Dragon
+0051 = Entered CCA
+0052 = Defeated Gal Gyrphon // Defeated Gol dragon in seat of heart (?)
+0054 = Entered Seabed Upper
+0057 = Defeated Olga Flow
+005B = Lab Natasha Start
+005C = Lab Natasha after VR temple
+005D = Lab Natasha after VR Spaceship
+005E = Lab Assistant after defeating Gal gryphon
+005F = After reading the last capsule from flowen
+0060 = Lab Natasha after CCA
+0065 = Cleared Magnitude of Metal
+0067 = Cleared Claiming a Stake
+0069 = Cleared Value of Money
+006B = Cleared Battle Training
+006D = Cleared Journalistic Pursuit
+006F = Cleared The Fake in Yellow
+0071 = Cleared Native Research
+0073 = Cleared Forest of Sorrow
+0075 = Cleared Gran Squall
+0077 = Cleared Addicting Food
+0079 = Cleared The Lost Bride
+007B = Cleared Waterfall Tears
+007D = Cleared Black Paper
+007F = Cleared Secret Delivery
+0081 = Cleared Soul of a Blacksmith
+0083 = Cleared Letter from Lionel
+0085 = Cleared The Grave's Butler
+0087 = Cleared Knowing One's Heart
+0089 = Cleared The Retired Hunter
+008B = Cleared Dr. Osto's Research
+008D = Cleared Unsealed Door
+008F = Cleared Soul of Steel
+0091 = Cleared Doc's Secret Plan (able to make enemy part weapons)
+0093 = Cleared Seek my Master
+0095 = Cleared From the Depths
+0096 = Unknown (set in the fake in yellow)
+0097 = Seat of heart unknown
+009B = Cleared Central Dome Fire Swirl
+00A1 = Cleared Seat of the Heart
+00C9 = Got an enemy weapon converted
+00CA = unknown Fake In Yellow
+00CE = unknown Fake In Yellow
+00D3 = Dr.Osto's research black paper subplot. Told Sue your name
+00D4 = Dr.Osto's research black paper subplot. Didn't tell Sue your name from before.
+00D5 = Dr.Osto's research black paper subplot. Did tell Sue your name from before.
+00D6 = Unsealed door. black paper subplot Talked to Sue. Refused to tell her your name
+00D7 = Unsealed door. black paper subplot. bernie tells you Sue is part of black paper.
+00D8 = Black paper subplot in waterfall of tears talking to Sue
+00D9 = Black paper subplot in Black paper talking to Sue (used option 2)
+00DB = Black paper subplot in Black paper talking to Sue (used any option)
+00DE = Black paper subplot in Black paper talked to Sue at the end of quest?
+00DF = Knowing ones heart talked to Bernie?
+00E0 = Seek my master. Zoke ,Donoph subplot?
+00E2 = Bernie Gran Squall
+00E7 = Defeated Kireek in waterfall of tears
+00E8 = Black paper subplot in black paper. defeated Kireek...
+00EB = Black paper subplot in from the depths. Defeated Kireek and got soul eater!
+00F1 = Secret delivery. Started the Weapons subplot //is cleared if quest is left
+00F3 = Weapon badge approval for claiming the snake //is cleared if quest is left
+00F4 = Weapon badge approval for the lost bride //is cleared if quest is left
+00F5 = Weapon badge approval for gran squall //is cleared if quest is left
+00F6 = Secret delivery. Got AKIKO's FRYING PAN!
+00FB = Got Orochi-agito
+00FB = Received OROCHI-AGITO!
+00FD = Unknown addicting food
+0105 = Central dome fire swirl. Got Glory of the past!
+0106 = Central dome fire swirl. Got Mark3.
+0107 = Central dome fire swirl. got Sonic knuckles
+0108 = Central dome fire swirl. got mail from BOGARDE
+0109 = Central dome fire swirl. got mail from ANNA
+010A = Central dome fire swirl. got mail from NADJA
+010B = Central dome fire swirl. got mail from Lionel
+010C = Soul of the blacksmith. Got one of the 3 special weapons!
+010D = Donoph Baz dies The Retired Hunter
+010E = Seat of heart unknown
+010F = Seat of heart unknown
+0110 = Seat of heart unknown
+0111 = Seat of heart unknown
+0112 = Seat of heart unknown
+0113 = Seat of heart unknown
+0187 = Soul of steel. Got Marina's bag! //dreamcast
+0188 = Soul of steel. Unknown.
+0191 = Capsule Elly VR
+0197 = Cleared VR Temple
+01AD = Capsule elly CCA
+01AE = Capsule elly CCA
+01B3 = After reading a capsule from flowen
+01D6 = Set after unlocking vr spaceship
+01F5 = Episode1: Cleared government 1-1
+01F7 = Episode1: Cleared government 1-2
+01F9 = Episode1: Cleared government 1-3
+01FB = Episode1: Cleared government 2-1
+01FD = Episode1: Cleared government 2-2
+01FF = Episode1: Cleared government 2-3
+0201 = Episode1: Cleared government 2-4
+0203 = Episode1: Cleared government 3-1
+0205 = Episode1: Cleared government 3-2
+0207 = Episode1: Cleared government 3-3
+0209 = Episode1: Cleared government 4-1
+020B = Episode1: Cleared government 4-2
+020D = Episode1: Cleared government 4-3
+020F = Episode1: Cleared government 4-4
+0211 = Episode1: Cleared government 4-5
+0213 = Episode2: Cleared government 5-1 // Talked to Tekker (?)
+0214 = Entered Forest 1
+0215 = Episode2: Cleared government 5-2
+0217 = Episode2: Cleared government 5-3 // Defeated Dragon (?)
+0219 = Episode2: Cleared government 5-4
+021B = Episode2: Cleared government 5-5
+021D = Episode2: Cleared government 6-1
+021F = Episode2: Cleared government 6-2
+0220 = Defeated De Rol Le
+0221 = Episode2: Cleared government 6-3
+0223 = Episode2: Cleared government 6-4
+0225 = Episode2: Cleared government 6-5
+0227 = Episode2: Cleared government 7-1
+0229 = Episode2: Cleared government 7-2
+022A = Defeated Vol Opt (002A and 022A together on hard mode)
+022B = Episode2: Cleared government 7-3 // Rico capsule after Vol Opt, at Ruins door (?)
+022D = Episode2: Cleared government 7-4 // Entered Caves 2 (?)
+022F = Episode2: Cleared government 7-5
+0230 = Entered Ruins 1
+0231 = Episode2: Cleared government 8-1
+0233 = Episode2: Cleared government 8-2
+0234 = Entered Falz 2
+0235 = Episode2: Cleared government 8-3
+0246 = Activated Jungle East big door switch
+0248 = Activated Seaside big door switch
+024F = Defeated Gol Dragon
+0252 = Defeated Gal Gryphon
+02BD = Episode4: Cleared government 9-1
+02BE = Episode4: Cleared government 9-2
+02BF = Episode4: Cleared government 9-3
+02C0 = Episode4: Cleared government 9-4
+02C1 = Episode4: Cleared government 9-5
+02C2 = Episode4: Cleared government 9-6
+02C3 = Episode4: Cleared government 9-7
+02C4 = Episode4: Cleared government 9-8
+0314 = Entered Forest 1
+0330 = Entered Ruins 1
+03FA = P2 Menu 7, G-Counter // Talked to Momoka
+03FB = Nol start
+03FC = Cleared Ep2 government on ultimate
+03FE = Cleared Ep2 government on normal-vh

notes/rare-drop-stream-example.py

@@ -0,0 +1,21 @@
+import asyncio
+import aiohttp
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        async with session.ws_connect("ws://localhost:5050/y/rare-drops/stream") as ws:
+            async for msg in ws:
+                if msg.type == aiohttp.WSMsgType.TEXT:
+                    data = msg.json()
+                    print(f"Received message: {data}")
+                elif msg.type == aiohttp.WSMsgType.BINARY:
+                    print(f"Received binary data: {msg.data}")
+                elif msg.type == aiohttp.WSMsgType.CLOSE:
+                    break
+                elif msg.type == aiohttp.WSMsgType.ERROR:
+                    break
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

notes/solo-quest-flags.txt

@@ -0,0 +1,27 @@
+###  T  FLAG  NAME                     REQUIREMENTS        AVAILABLE_IF                          ENABLED_IF
+001  1  0065  Magnitude of Metal                                                                 !F_0065 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+002  1  0067  Claiming A Stake                                                                   !F_0067 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+003  3  0069  The Value of Money       T1, Caves           F_0065 && F_0067 && F_006B && F_01F9  !F_0069 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+004  1  006B  Battle Training                                                                    !F_006B || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+005  2  006D  Journalistic Pursuit     T1                  F_0065 && F_0067 && F_006B            !F_006D || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+006  2  006F  The Fake in yellow       T1                  F_0065 && F_0067 && F_006B            !F_006F || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+007  2  0071  Native Research          T1                  F_0065 && F_0067 && F_006B            !F_0071 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+008  2  0073  Forest of Sorrow         007                 F_0071                                !F_0073 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+009  2  0075  Gran Squall              T1                  F_0065 && F_0067 && F_006B            !F_0075 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+010  3  0077  Addicting Food           T1                  F_0065 && F_0067 && F_006B            !F_0077 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+011  3  0079  The Lost Bride           T1                  F_0065 && F_0067 && F_006B            !F_0079 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+012  3  007B  Waterfall tears          010, 011, 014, 017  F_0077 && F_0079 && F_007F && F_0085  !F_007B || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+013  3  007D  Black Paper              012                 F_007B                                !F_007D || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+014  3  007F  Secret Delivery          T1                  F_0065 && F_0067 && F_006B            !F_007F || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+015  3  0081  Soul of a Blacksmith     010, 011, 014, 017  F_0077 && F_0079 && F_007F && F_0085  !F_0081 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+016  3  0083  Letter from Lionel       T1, Mines           F_0065 && F_0067 && F_006B && F_0201  !F_0083 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+017  3  0085  The Grave's Butler       T1                  F_0065 && F_0067 && F_006B            !F_0085 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+018  4  0087  Knowing One's Heart      T1, Mines           F_0065 && F_0067 && F_006B && F_0201  !F_0087 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+019  5  0089  Retired Hunter           T1, Ruins           F_0065 && F_0067 && F_006B && F_0207  !F_0089 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+020  4  008B  Dr. Osto's Research      T1, Mines           F_0065 && F_0067 && F_006B && F_0201  !F_008B || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+021  4  008D  The Unsealed Door        020, 014            F_008B && F_007F                      !F_008D || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+022  5  008F  Soul of Steel            023                 F_0091                                !F_008F || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+023  5  0091  Doc's Secret Plan        014, Ruins          F_007F && F_0207                      !F_0091 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+024  5  0093  Seek My Master           T1, Ruins           F_0065 && F_0067 && F_006B && F_0207  !F_0093 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+025  5  0095  From the Depths          023                 F_0091                                !F_0095 || (F_0065 && F_0067 && F_0069 && F_006B && F_006D && F_006F && F_0071 && F_0073 && F_0075 && F_0077 && F_0079 && F_007B && F_007D && F_007F && F_0081 && F_0083 && F_0085 && F_0087 && F_0089 && F_008B && F_008D && F_008F && F_0091 && F_0093 && F_0095)
+026  2  009B  Central Dome Fire Swirl  008                 F_0073

notes/star-value-tables.txt

@@ -1,11 +0,0 @@
-star value tables
-
-psobb [B1-437]
-00010203 04090909 01020304 05090909 01020304 05090909 01020304 05090909 01020304 05090909 00010203 04090909 01020304 05090909 01020304 05090909 01020304 05090909 00010203 09090901 02030409 09090102 03040909 09090A0A 090A0A09 0A0A090C 0B0A0A0A 0A0A0A0B 0A090A0A 0A0A0A09 0A0A0A0A 0A0A0A0A 0A0A0B0A 0C0C0B0A 0A090A09 090A0A0A 0A0C090C 0B0A090A 090C0A0B 0A0A0A0A 0A0A0A0B 0B0A0A0A 09090A09 0C0A0A0A 0B0A0B09 0A0A090A 0A0B090B 0A0B0B0A 090A090A 0B090A0A 0A0A0A0A 0A0A0A09 090C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0009 0A0A0A0B 090A0A09 0A0A0B0A 0A0A0A0A 0A0A0A0A 0A0A0A0A 0A0A0A0A 0A0A0A0A 0B0B0B0B 0B0B0B0B 0B0B0B0A 0C0A0C0B 0A0A0A0A 0A0B0A0B 0B0B0B0B 0A0A090A 0A0A090B 0B0B0B0C 0C0C0C0C 0A0A0C0A 090A0C09 0A0B0A0A 0A0A0C0A 0A0A0A09 0A0C0A09 0A0A0A0A 0A090C0B 09090909 09090909 09090909 09090909 0909090B 0A0C0A0B 0B0C0A0A 0A090A0A 0A0A0B0A 0A0A0A0A 0909090A 0A090C0A 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0102 03040102 03040203 04020304 01020304 01020304 01020304 01020304 01020304 01020304 03040000 00010102 02030304 04050506 06070707 07080808 08080809 09090A0A 0A0A0A0A 0B0C0A0A 0A0A0A0A 0B0B0B0A 0B0B0C0B 0B0B0B0B 0A0A0A0A 0A0A0C09 0909090A 0A0B0C09 0B0A0A0A 0A0A0A0A 0A0A0A0B 0A0A0A0A 0A0A0A00 00010203 03040405 05050606 07070808 08080808 0A0A0A0A 0A0A0909 090A0A0A 0A0A0A0A 0A0A0B0A 0A0B0A09 0909090A 0B0B0000 0B000000 00080808 08080808 09080808 09080808 09070707 07070909 090C0909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 0A0A0B0A 0B0A0909 0B0B0B0C 0A0A0A09 0A0A0A0A 090A0A0A 0A0A0A0A 0A0A0A0A 0203050B 0203050B 0203050B 0203050B 0204060B 0204060B 0203050B 080B080A 0B020305 02030502 03050304 06030405 07080B04 06090406 09040609 06090B06 090B0909 09090909 0A0B0B0B 0B0B0B0B 0B0B0B0B 0B0B0B0B 0B0B0B0B 0A0B0B0B 0B0B0B0B
-
-psogc [94-2F7]
-00010203 04090909 01020304 05090909 01020304 05090909 01020304 05090909 01020304 05090909 00010203 04090909 01020304 05090909 01020304 05090909 01020304 05090909 00010203 09090901 02030409 09090102 03040909 09090A0A 090A0A09 0A0A090C 0B0A0A0A 0A0A0A0B 0A090A0A 0A0A0A09 0A0A0A0A 0A0A0A0A 0A0A0B0A 0C0C0B0A 0A090A09 090A0A0A 0A0C090C 0B0A090A 090C0A0B 0A0A0A0A 0A0A0A0B 0B0A0A0A 09090A09 0C0A0A0A 0B0A0B09 0A0A090A 0A0B090B 0A0B0B0A 090A090A 0B090A0A 0A0A0A0A 0A0A0A09 090C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0C0C 0C0C0009 0A0A0A0B 090A0A09 0A0A0B0A 0A0A0A0A 0A0A0A0A 0A0A0A0A 0A0A0A0A 0A0A0A0A 0B0B0B0B 0B0B0B0B 0B0B0B0A 0C0A0C0B 0A0A0A0A 0A0B0A0B 0B0B0B0B 0A0A090A 0A0A090B 0B0B0B0C 0C0C0C0C 01020304 01020304 02030402 03040102 03040102 03040102 03040102 03040102 03040102 03040304 00000001 01020203 03040405 05060607 07070708 08080808 08090909 0A0A0A0A 0A0A0B0C 0A0A0A0A 0A0A0B0B 0B0A0B0B 0C0B0B0B 0B0B0000 01020303 04040505 05060607 07080808 0808080A 0A0A0A0A 0A090909 0A0A0A0A 0A0A0A0A 0A0B0A0A 0B0A0909 09090A0B 0B000000 00000000 08080808 08080809 08080809 08080809 07070707 07090909 0C090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090909 09090902 03050B02 03050B02 03050B02 03050B02 04060B02 04060B02 03050B08 0B080A0B 02030502 03050203 05030406 03040507 080B0406 09040609 04060906 090B0609 0B090909 090909
-
-
-
-0203050B0203050B0203050B0203050B

src/AFSArchive.cc

@@ -7,6 +7,8 @@
 #include <phosg/Filesystem.hh>
 #include <phosg/Strings.hh>
 
+#include "Text.hh"
+
 using namespace std;
 
 AFSArchive::AFSArchive(shared_ptr<const string> data)
@@ -14,14 +16,14 @@ AFSArchive::AFSArchive(shared_ptr<const string> data)
   struct FileHeader {
     be_uint32_t magic;
     le_uint32_t num_files;
-  } __attribute__((packed));
+  } __packed_ws__(FileHeader, 8);
 
   struct FileEntry {
     le_uint32_t offset;
     le_uint32_t size;
-  } __attribute__((packed));
+  } __packed_ws__(FileEntry, 8);
 
-  StringReader r(*this->data);
+  phosg::StringReader r(*this->data);
   const auto& header = r.get<FileHeader>();
   if (header.magic != 0x41465300) { // 'AFS\0'
     throw runtime_error("file is not an AFS archive");
@@ -50,29 +52,27 @@ string AFSArchive::get_copy(size_t index) const {
   return string(reinterpret_cast<const char*>(ret.first), ret.second);
 }
 
-StringReader AFSArchive::get_reader(size_t index) const {
+phosg::StringReader AFSArchive::get_reader(size_t index) const {
   auto ret = this->get(index);
-  return StringReader(ret.first, ret.second);
+  return phosg::StringReader(ret.first, ret.second);
 }
 
 string AFSArchive::generate(const vector<string>& files, bool big_endian) {
   return big_endian ? AFSArchive::generate_t<true>(files) : AFSArchive::generate_t<false>(files);
 }
 
-template <bool IsBigEndian>
+template <bool BE>
 string AFSArchive::generate_t(const vector<string>& files) {
-  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
-
-  StringWriter w;
+  phosg::StringWriter w;
   w.put_u32b(0x41465300); // 'AFS\0'
-  w.put<U32T>(files.size());
+  w.put<U32T<BE>>(files.size());
 
   // It seems entries are aligned to 0x800-byte boundaries, and the file's
   // header is always 0x80000 (!) bytes, most of which is unused
   uint32_t data_offset = 0x80000;
   for (const auto& file : files) {
-    w.put<U32T>(data_offset);
-    w.put<U32T>(file.size());
+    w.put<U32T<BE>>(data_offset);
+    w.put<U32T<BE>>(file.size());
     data_offset = (data_offset + file.size() + 0x7FF) & (~0x7FF);
   }
 

src/AFSArchive.hh

@@ -8,6 +8,8 @@
 #include <string>
 #include <unordered_map>
 
+#include "Types.hh"
+
 class AFSArchive {
 public:
   AFSArchive(std::shared_ptr<const std::string> data);
@@ -23,12 +25,12 @@ public:
 
   std::pair<const void*, size_t> get(size_t index) const;
   std::string get_copy(size_t index) const;
-  StringReader get_reader(size_t index) const;
+  phosg::StringReader get_reader(size_t index) const;
 
   static std::string generate(const std::vector<std::string>& files, bool big_endian);
 
 private:
-  template <bool IsBigEndian>
+  template <bool BE>
   static std::string generate_t(const std::vector<std::string>& files);
 
   std::shared_ptr<const std::string> data;

src/ARCodeTranslator-Stub.hh

@@ -1,14 +0,0 @@
-#pragma once
-
-#include <stdexcept>
-#include <string>
-#include <utility>
-#include <vector>
-
-inline void run_ar_code_translator(const std::string&, const std::string&, const std::string&) {
-  throw std::runtime_error("resource_file is not available; install it and rebuild newserv");
-}
-
-inline std::vector<std::pair<uint32_t, std::string>> diff_dol_files(const std::string&, const std::string&) {
-  throw std::runtime_error("resource_file is not available; install it and rebuild newserv");
-}

src/ARCodeTranslator.cc

@@ -1,401 +0,0 @@
-#include "ARCodeTranslator.hh"
-
-#include <array>
-#include <future>
-#include <phosg/Filesystem.hh>
-#include <phosg/Strings.hh>
-#include <resource_file/ExecutableFormats/DOLFile.hh>
-
-using namespace std;
-
-class ARCodeTranslator {
-public:
-  enum class ExpandMethod {
-    FORWARD = 0,
-    FORWARD_WITH_BARRIER,
-    BACKWARD,
-    BACKWARD_WITH_BARRIER,
-    BOTH,
-    BOTH_WITH_BARRIER,
-    BOTH_IGNORE_ORIGIN,
-  };
-
-  static const char* name_for_expand_method(ExpandMethod method) {
-    switch (method) {
-      case ExpandMethod::FORWARD:
-        return "FORWARD";
-      case ExpandMethod::FORWARD_WITH_BARRIER:
-        return "FORWARD_WITH_BARRIER";
-      case ExpandMethod::BACKWARD:
-        return "BACKWARD";
-      case ExpandMethod::BACKWARD_WITH_BARRIER:
-        return "BACKWARD_WITH_BARRIER";
-      case ExpandMethod::BOTH:
-        return "BOTH";
-      case ExpandMethod::BOTH_WITH_BARRIER:
-        return "BOTH_WITH_BARRIER";
-      case ExpandMethod::BOTH_IGNORE_ORIGIN:
-        return "BOTH_IGNORE_ORIGIN";
-      default:
-        throw logic_error("invalid expand method");
-    }
-  }
-
-  ARCodeTranslator(const string& directory)
-      : log("[ar-trans] "),
-        directory(directory) {
-    while (ends_with(this->directory, "/")) {
-      this->directory.pop_back();
-    }
-    for (const auto& filename : list_directory(this->directory)) {
-      if (ends_with(filename, ".dol")) {
-        string name = filename.substr(0, filename.size() - 4);
-        string path = directory + "/" + filename;
-        this->files.emplace(name, make_shared<DOLFile>(path.c_str()));
-        this->log.info("Loaded %s", name.c_str());
-      }
-    }
-  }
-  ~ARCodeTranslator() = default;
-
-  const string& get_source_filename() const {
-    return this->src_filename;
-  }
-  void set_source_file(const string& filename) {
-    this->src_filename = filename;
-    this->src_file = files.at(this->src_filename);
-  }
-
-  void find_rtoc_global_regs() const {
-    for (const auto& it : files) {
-      bool r2_high_found = false;
-      bool r2_low_found = false;
-      bool r13_high_found = false;
-      bool r13_low_found = false;
-      uint32_t r2 = 0;
-      uint32_t r13 = 0;
-      for (const auto& section : it.second->sections) {
-        if (!section.is_text) {
-          continue;
-        }
-        StringReader r(section.data);
-        while (!r.eof() && r.where()) {
-          uint32_t opcode = r.get_u32b();
-          if ((opcode & 0xFFFF0000) == 0x3DA00000) {
-            if (r13_high_found) {
-              throw runtime_error("multiple values for r13_high");
-            }
-            r13_high_found = true;
-            r13 |= (opcode << 16);
-          } else if ((opcode & 0xFFFF0000) == 0x3C400000) {
-            if (r2_high_found) {
-              throw runtime_error("multiple values for r2_high");
-            }
-            r2_high_found = true;
-            r2 |= (opcode << 16);
-          } else if ((opcode & 0xFFFF0000) == 0x61AD0000) {
-            if (r13_low_found) {
-              throw runtime_error("multiple values for r13_low");
-            }
-            r13_low_found = true;
-            r13 |= (opcode & 0xFFFF);
-          } else if ((opcode & 0xFFFF0000) == 0x60420000) {
-            if (r2_low_found) {
-              throw runtime_error("multiple values for r2_low");
-            }
-            r2_low_found = true;
-            r2 |= (opcode & 0xFFFF);
-          }
-        }
-      }
-      if (r2_low_found && r2_high_found) {
-        fprintf(stderr, "(%s) r2 = %08" PRIX32 "\n", it.first.c_str(), r2);
-      } else {
-        fprintf(stderr, "(%s) r2 = __MISSING__\n", it.first.c_str());
-      }
-      if (r13_low_found && r13_high_found) {
-        fprintf(stderr, "(%s) r13 = %08" PRIX32 "\n", it.first.c_str(), r13);
-      } else {
-        fprintf(stderr, "(%s) r13 = __MISSING__\n", it.first.c_str());
-      }
-    }
-  }
-
-  uint32_t find_match(shared_ptr<const DOLFile> dest_file, uint32_t src_address, ExpandMethod expand_method) const {
-    if (!this->src_file) {
-      throw runtime_error("no source file selected");
-    }
-
-    const DOLFile::Section* src_section = nullptr;
-    for (const auto& sec : this->src_file->sections) {
-      if (src_address >= sec.address && src_address < sec.address + sec.data.size()) {
-        src_section = &sec;
-        break;
-      }
-    }
-    if (!src_section) {
-      throw runtime_error("source address not within any section");
-    }
-
-    const char* method_token = this->name_for_expand_method(expand_method);
-
-    size_t src_offset = src_address - src_section->address;
-    size_t src_bytes_available_before = src_offset;
-    size_t src_bytes_available_after = src_section->data.size() - src_offset - 4;
-    this->log.info("(find_match/%s) Source offset = %08zX with %zX/%zX bytes available before/after",
-        method_token, src_offset, src_bytes_available_before, src_bytes_available_after);
-
-    size_t match_bytes_before = 0;
-    size_t match_bytes_after = 0;
-    while (match_bytes_before + match_bytes_after + 4 < 0x100) {
-      size_t num_matches = 0;
-      size_t last_match_address = 0;
-      size_t match_length = match_bytes_before + match_bytes_after + 4;
-      StringReader src_r(src_section->data.data() + src_offset - match_bytes_before, match_length);
-      for (const auto& dest_section : dest_file->sections) {
-        for (size_t dest_match_offset = 0;
-             dest_match_offset < dest_section.data.size();
-             dest_match_offset += 4) {
-          src_r.go(0);
-          StringReader dest_r(dest_section.data.data() + dest_match_offset, match_length);
-          size_t z;
-          for (z = 0; z < match_length; z += 4) {
-            if (expand_method == ExpandMethod::BOTH_IGNORE_ORIGIN && z == match_bytes_before) {
-              src_r.skip(4);
-              dest_r.skip(4);
-            } else if (src_section->is_text) {
-              uint32_t src_opcode = src_r.get_u32b();
-              uint32_t dest_opcode = dest_r.get_u32b();
-              uint32_t src_class = src_opcode & 0xFC000000;
-              if (src_class != (dest_opcode & 0xFC000000)) {
-                break;
-              }
-              if (src_class == 0x48000000) {
-                // b +-offset
-                src_opcode &= 0xFC000003;
-                dest_opcode &= 0xFC000003;
-              } else if (((src_opcode & 0xAC1F0000) == 0x800D0000) || ((src_opcode & 0xAC1F0000) == 0x80020000)) {
-                // lwz/lfs rXX/fXX, [r2/r13 +- offset] OR stw/stfs [r2/r13 +- offset], rXX/fXX
-                src_opcode &= 0xFFFF0000;
-                dest_opcode &= 0xFFFF0000;
-              }
-              if (src_opcode != dest_opcode) {
-                break;
-              }
-            } else {
-              uint32_t src_data = src_r.get_u32b();
-              uint32_t dest_data = dest_r.get_u32b();
-              if ((src_data & 0xFE000000) == 0x80000000) {
-                src_data &= 0xFE000003;
-              }
-              if ((dest_data & 0xFE000000) == 0x80000000) {
-                dest_data &= 0xFE000003;
-              }
-              if (src_data != dest_data) {
-                break;
-              }
-            }
-          }
-          if (z == match_length) {
-            num_matches++;
-            last_match_address = dest_section.address + dest_match_offset + match_bytes_before;
-          }
-        }
-      }
-      this->log.info("(find_match/%s) For match length %zX, %zu matches found", method_token, match_length, num_matches);
-      if (num_matches == 1) {
-        return last_match_address;
-      } else if (num_matches == 0) {
-        throw runtime_error("did not find exactly one match");
-      }
-      bool can_expand_backward = false;
-      bool can_expand_forward = false;
-      switch (expand_method) {
-        case ExpandMethod::BACKWARD_WITH_BARRIER:
-          can_expand_backward = (src_r.pget_u32b(0) != 0x4E800020) &&
-              (src_bytes_available_before >= match_bytes_before + 4);
-          break;
-        case ExpandMethod::BACKWARD:
-          can_expand_backward = (src_bytes_available_before >= match_bytes_before + 4);
-          break;
-        case ExpandMethod::FORWARD_WITH_BARRIER:
-          can_expand_forward = (src_r.pget_u32b(src_r.size() - 4) != 0x4E800020) &&
-              (src_bytes_available_after >= match_bytes_after + 4);
-          break;
-        case ExpandMethod::FORWARD:
-          can_expand_forward = (src_bytes_available_after >= match_bytes_after + 4);
-          break;
-        case ExpandMethod::BOTH_WITH_BARRIER:
-        case ExpandMethod::BOTH_IGNORE_ORIGIN:
-          can_expand_backward = (src_r.pget_u32b(0) != 0x4E800020) &&
-              (src_bytes_available_before >= match_bytes_before + 4);
-          can_expand_forward = (src_r.pget_u32b(src_r.size() - 4) != 0x4E800020) &&
-              (src_bytes_available_after >= match_bytes_after + 4);
-          break;
-        case ExpandMethod::BOTH:
-          can_expand_backward = (src_bytes_available_before >= match_bytes_before + 4);
-          can_expand_forward = (src_bytes_available_after >= match_bytes_after + 4);
-          break;
-        default:
-          throw logic_error("invalid expand method");
-      }
-      if (!can_expand_backward && !can_expand_forward) {
-        throw runtime_error("no further expansion is allowed");
-      }
-      if (can_expand_backward) {
-        match_bytes_before += 4;
-      }
-      if (can_expand_forward) {
-        match_bytes_after += 4;
-      }
-    }
-    throw runtime_error("scan field too long; too many matches");
-  }
-
-  void find_all_matches(uint32_t src_addr) const {
-    if (!this->src_file) {
-      throw runtime_error("no source file selected");
-    }
-
-    unordered_map<string, uint32_t> results;
-    for (const auto& it : files) {
-      if (it.second == this->src_file) {
-        log.info("(%s) %08" PRIX32 " (from source)", it.first.c_str(), src_addr);
-        results.emplace(it.first, src_addr);
-      } else {
-
-        array<future<uint32_t>, 7> futures;
-        static const array<ExpandMethod, 7> methods = {
-            ExpandMethod::FORWARD,
-            ExpandMethod::FORWARD_WITH_BARRIER,
-            ExpandMethod::BACKWARD,
-            ExpandMethod::BACKWARD_WITH_BARRIER,
-            ExpandMethod::BOTH,
-            ExpandMethod::BOTH_WITH_BARRIER,
-            ExpandMethod::BOTH_IGNORE_ORIGIN,
-        };
-        for (size_t z = 0; z < methods.size(); z++) {
-          futures[z] = async(&ARCodeTranslator::find_match, this, it.second, src_addr, methods[z]);
-        }
-
-        unordered_set<uint32_t> match_addrs;
-        for (size_t z = 0; z < futures.size(); z++) {
-          const char* method_name = this->name_for_expand_method(methods[z]);
-          try {
-            uint32_t ret = futures[z].get();
-            log.info("(%s) (%s) %08" PRIX32, it.first.c_str(), method_name, ret);
-            match_addrs.emplace(ret);
-          } catch (const exception& e) {
-            log.error("(%s) (%s) failed: %s", it.first.c_str(), method_name, e.what());
-          }
-        }
-
-        if (match_addrs.empty()) {
-          log.error("(%s) no match found", it.first.c_str());
-        } else if (match_addrs.size() > 1) {
-          log.error("(%s) different matches found by different methods", it.first.c_str());
-        } else {
-          results.emplace(it.first, *match_addrs.begin());
-        }
-      }
-    }
-    for (const auto& it : results) {
-      fprintf(stdout, "%s => %08" PRIX32 "\n", it.first.c_str(), it.second);
-    }
-  }
-
-  void handle_command(const string& command) {
-    auto tokens = split(command, ' ');
-    if (tokens.empty()) {
-      throw runtime_error("no command given");
-    }
-    strip_trailing_whitespace(tokens[tokens.size() - 1]);
-
-    if (tokens[0] == "use") {
-      this->set_source_file(tokens.at(1));
-    } else if (tokens[0] == "match") {
-      this->find_all_matches(stoul(tokens.at(1), nullptr, 16));
-    } else if (tokens[0] == "find-globals") {
-      this->find_rtoc_global_regs();
-    } else if (!tokens[0].empty()) {
-      throw runtime_error("unknown command");
-    }
-  }
-
-  void run_shell() {
-    while (!feof(stdin)) {
-      if (!this->src_filename.empty()) {
-        fprintf(stdout, "ar-trans:%s/%s> ", this->directory.c_str(), this->src_filename.c_str());
-      } else {
-        fprintf(stdout, "ar-trans:%s> ", this->directory.c_str());
-      }
-      fflush(stdout);
-
-      string command = fgets(stdin);
-      try {
-        this->handle_command(command);
-      } catch (const exception& e) {
-        this->log.error("Failed: %s", e.what());
-      }
-    }
-    fputc('\n', stdout);
-  }
-
-private:
-  PrefixedLogger log;
-  string directory;
-  unordered_map<string, shared_ptr<const DOLFile>> files;
-  string src_filename;
-  shared_ptr<const DOLFile> src_file;
-};
-
-void run_ar_code_translator(const std::string& directory, const std::string& use_filename, const std::string& command) {
-  ARCodeTranslator trans(directory);
-  if (!use_filename.empty()) {
-    trans.set_source_file(use_filename);
-  }
-
-  if (!command.empty()) {
-    trans.handle_command(command);
-  } else {
-    trans.run_shell();
-  }
-}
-
-vector<pair<uint32_t, string>> diff_dol_files(const string& a_filename, const string& b_filename) {
-  DOLFile a(a_filename.c_str());
-  DOLFile b(b_filename.c_str());
-  auto a_mem = make_shared<MemoryContext>();
-  auto b_mem = make_shared<MemoryContext>();
-  a.load_into(a_mem);
-  b.load_into(b_mem);
-
-  uint32_t min_addr = 0xFFFFFFFF;
-  uint32_t max_addr = 0x00000000;
-  for (const auto& sec : a.sections) {
-    min_addr = min<uint32_t>(min_addr, sec.address);
-    max_addr = max<uint32_t>(max_addr, sec.address + sec.data.size());
-  }
-  for (const auto& sec : b.sections) {
-    min_addr = min<uint32_t>(min_addr, sec.address);
-    max_addr = max<uint32_t>(max_addr, sec.address + sec.data.size());
-  }
-
-  vector<pair<uint32_t, string>> ret;
-  for (uint32_t addr = min_addr; addr < max_addr; addr += 4) {
-    bool a_exists = a_mem->exists(addr, 4);
-    bool b_exists = b_mem->exists(addr, 4);
-    if (a_exists && b_exists) {
-      string a_value = a_mem->read(addr, 4);
-      string b_value = b_mem->read(addr, 4);
-      if (a_value != b_value) {
-        if (!ret.empty() && (ret.back().first + ret.back().second.size() == addr)) {
-          ret.back().second += b_value;
-        } else {
-          ret.emplace_back(make_pair(addr, b_value));
-        }
-      }
-    }
-  }
-  return ret;
-}

src/ARCodeTranslator.hh

@@ -1,10 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-
-#include <string>
-#include <utility>
-#include <vector>
-
-void run_ar_code_translator(const std::string& initial_directory, const std::string& use_file, const std::string& command);
-std::vector<std::pair<uint32_t, std::string>> diff_dol_files(const std::string& a_filename, const std::string& b_filename);

src/Account.hh

@@ -0,0 +1,284 @@
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <phosg/JSON.hh>
+#include <shared_mutex>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "Text.hh"
+
+class LicenseIndex;
+
+struct DCNTELicense {
+  std::string serial_number;
+  std::string access_key;
+
+  static std::shared_ptr<DCNTELicense> from_json(const phosg::JSON& json);
+  phosg::JSON json() const;
+};
+
+struct V1V2License {
+  uint32_t serial_number = 0;
+  std::string access_key;
+
+  static std::shared_ptr<V1V2License> from_json(const phosg::JSON& json);
+  phosg::JSON json() const;
+};
+
+struct GCLicense {
+  uint32_t serial_number = 0;
+  std::string access_key;
+  std::string password;
+
+  static std::shared_ptr<GCLicense> from_json(const phosg::JSON& json);
+  phosg::JSON json() const;
+};
+
+struct XBLicense {
+  std::string gamertag;
+  uint64_t user_id = 0;
+  uint64_t account_id = 0;
+
+  static std::shared_ptr<XBLicense> from_json(const phosg::JSON& json);
+  phosg::JSON json() const;
+};
+
+struct BBLicense {
+  std::string username;
+  std::string password;
+
+  static std::shared_ptr<BBLicense> from_json(const phosg::JSON& json);
+  phosg::JSON json() const;
+};
+
+struct Account {
+  enum class Flag : uint32_t {
+    // clang-format off
+    KICK_USER                   = 0x00000001,
+    BAN_USER                    = 0x00000002,
+    SILENCE_USER                = 0x00000004,
+    CHANGE_EVENT                = 0x00000010,
+    ANNOUNCE                    = 0x00000020,
+    FREE_JOIN_GAMES             = 0x00000040,
+    DEBUG                       = 0x01000000,
+    CHEAT_ANYWHERE              = 0x02000000,
+    DISABLE_QUEST_REQUIREMENTS  = 0x04000000,
+    ALWAYS_ENABLE_CHAT_COMMANDS = 0x08000000,
+    MODERATOR                   = 0x00000007,
+    ADMINISTRATOR               = 0x000000FF,
+    ROOT                        = 0x7FFFFFFF,
+    IS_SHARED_ACCOUNT           = 0x80000000,
+    // NOTE: When adding or changing license flags, don't forget to change the
+    // documentation in the shell's help text.
+    UNUSED_BITS                 = 0x70FFFF00,
+    // clang-format on
+  };
+  enum class UserFlag : uint32_t {
+    DISABLE_DROP_NOTIFICATION_BROADCAST = 0x00000001,
+  };
+
+  // account_id is also the account's guild card number
+  uint32_t account_id = 0;
+
+  uint32_t flags = 0;
+  uint32_t user_flags = 0;
+  uint64_t ban_end_time = 0; // 0 = not banned
+  std::string last_player_name;
+  std::string auto_reply_message;
+
+  uint32_t ep3_current_meseta = 0;
+  uint32_t ep3_total_meseta_earned = 0;
+
+  uint32_t bb_team_id = 0;
+  bool is_temporary = false; // If true, isn't saved to disk
+
+  std::unordered_set<std::string> auto_patches_enabled;
+
+  std::unordered_map<std::string, std::shared_ptr<DCNTELicense>> dc_nte_licenses;
+  std::unordered_map<uint32_t, std::shared_ptr<V1V2License>> dc_licenses;
+  std::unordered_map<uint32_t, std::shared_ptr<V1V2License>> pc_licenses;
+  std::unordered_map<uint32_t, std::shared_ptr<GCLicense>> gc_licenses;
+  std::unordered_map<std::string, std::shared_ptr<XBLicense>> xb_licenses;
+  std::unordered_map<std::string, std::shared_ptr<BBLicense>> bb_licenses;
+
+  Account() = default;
+  explicit Account(const phosg::JSON& json);
+  virtual ~Account() = default;
+
+  phosg::JSON json() const;
+  virtual void save() const;
+  virtual void delete_file() const;
+
+  [[nodiscard]] inline bool check_flag(Flag flag) const {
+    return !!(this->flags & static_cast<uint32_t>(flag));
+  }
+  inline void set_flag(Flag flag) {
+    this->flags |= static_cast<uint32_t>(flag);
+  }
+  inline void clear_flag(Flag flag) {
+    this->flags &= (~static_cast<uint32_t>(flag));
+  }
+  inline void toggle_flag(Flag flag) {
+    this->flags ^= static_cast<uint32_t>(flag);
+  }
+  inline void replace_all_flags(Flag mask) {
+    this->flags = static_cast<uint32_t>(mask);
+  }
+
+  [[nodiscard]] inline bool check_user_flag(UserFlag flag) const {
+    return !!(this->user_flags & static_cast<uint32_t>(flag));
+  }
+  inline void set_user_flag(UserFlag flag) {
+    this->user_flags |= static_cast<uint32_t>(flag);
+  }
+  inline void clear_user_flag(UserFlag flag) {
+    this->user_flags &= (~static_cast<uint32_t>(flag));
+  }
+  inline void toggle_user_flag(UserFlag flag) {
+    this->user_flags ^= static_cast<uint32_t>(flag);
+  }
+
+  void print(FILE* stream) const;
+};
+
+struct Login {
+  bool account_was_created = false;
+  // This field will never be null
+  std::shared_ptr<Account> account;
+  // Exactly one of the following will be non-null, representing the license
+  // that the client logged in with
+  std::shared_ptr<DCNTELicense> dc_nte_license;
+  std::shared_ptr<V1V2License> dc_license;
+  std::shared_ptr<V1V2License> pc_license;
+  std::shared_ptr<GCLicense> gc_license;
+  std::shared_ptr<XBLicense> xb_license;
+  std::shared_ptr<BBLicense> bb_license;
+};
+
+class AccountIndex {
+public:
+  class no_username : public std::invalid_argument {
+  public:
+    no_username() : invalid_argument("serial number is zero or username is missing") {}
+  };
+  class incorrect_password : public std::invalid_argument {
+  public:
+    incorrect_password() : invalid_argument("incorrect password") {}
+  };
+  class incorrect_access_key : public std::invalid_argument {
+  public:
+    incorrect_access_key() : invalid_argument("incorrect access key") {}
+  };
+  class missing_account : public std::invalid_argument {
+  public:
+    missing_account() : invalid_argument("missing account") {}
+  };
+  class account_banned : public std::invalid_argument {
+  public:
+    account_banned() : invalid_argument("account is banned") {}
+  };
+
+  explicit AccountIndex(bool force_all_temporary);
+  virtual ~AccountIndex() = default;
+
+  std::shared_ptr<Account> create_account(bool is_temporary) const;
+
+  size_t count() const;
+  std::vector<std::shared_ptr<Account>> all() const;
+
+  void add(std::shared_ptr<Account> a);
+  void remove(uint32_t serial_number);
+
+  void add_dc_nte_license(std::shared_ptr<Account> account, std::shared_ptr<DCNTELicense> license);
+  void add_dc_license(std::shared_ptr<Account> account, std::shared_ptr<V1V2License> license);
+  void add_pc_license(std::shared_ptr<Account> account, std::shared_ptr<V1V2License> license);
+  void add_gc_license(std::shared_ptr<Account> account, std::shared_ptr<GCLicense> license);
+  void add_xb_license(std::shared_ptr<Account> account, std::shared_ptr<XBLicense> license);
+  void add_bb_license(std::shared_ptr<Account> account, std::shared_ptr<BBLicense> license);
+  void remove_dc_nte_license(std::shared_ptr<Account> account, const std::string& serial_number);
+  void remove_dc_license(std::shared_ptr<Account> account, uint32_t serial_number);
+  void remove_pc_license(std::shared_ptr<Account> account, uint32_t serial_number);
+  void remove_gc_license(std::shared_ptr<Account> account, uint32_t serial_number);
+  void remove_xb_license(std::shared_ptr<Account> account, const std::string& gamertag);
+  void remove_bb_license(std::shared_ptr<Account> account, const std::string& username);
+
+  std::shared_ptr<Account> from_account_id(uint32_t account_id) const;
+  std::shared_ptr<Login> from_dc_nte_credentials(
+      const std::string& serial_number,
+      const std::string& access_key,
+      bool allow_create);
+  std::shared_ptr<Login> from_dc_credentials(
+      uint32_t serial_number,
+      const std::string& access_key,
+      const std::string& character_name,
+      bool allow_create);
+  std::shared_ptr<Login> from_pc_nte_credentials(
+      uint32_t guild_card_number,
+      bool allow_create);
+  std::shared_ptr<Login> from_pc_credentials(
+      uint32_t serial_number,
+      const std::string& access_key,
+      const std::string& character_name,
+      bool allow_create);
+  std::shared_ptr<Login> from_gc_credentials(
+      uint32_t serial_number,
+      const std::string& access_key,
+      const std::string* password,
+      const std::string& character_name,
+      bool allow_create);
+  std::shared_ptr<Login> from_xb_credentials(
+      const std::string& gamertag,
+      uint64_t user_id,
+      uint64_t account_id,
+      bool allow_create);
+  std::shared_ptr<Login> from_bb_credentials(
+      const std::string& username,
+      const std::string* password,
+      bool allow_create);
+
+  std::shared_ptr<Account> create_temporary_account_for_shared_account(
+      std::shared_ptr<const Account> src_a, const std::string& variation_data) const;
+
+protected:
+  bool force_all_temporary;
+
+  // This class must be thread-safe because it's used by both the patch server
+  // and game server threads
+  mutable std::shared_mutex lock;
+  std::unordered_map<uint32_t, std::shared_ptr<Account>> by_account_id;
+  std::unordered_map<std::string, std::shared_ptr<Account>> by_dc_nte_serial_number;
+  std::unordered_map<uint32_t, std::shared_ptr<Account>> by_dc_serial_number;
+  std::unordered_map<uint32_t, std::shared_ptr<Account>> by_pc_serial_number;
+  std::unordered_map<uint32_t, std::shared_ptr<Account>> by_gc_serial_number;
+  std::unordered_map<std::string, std::shared_ptr<Account>> by_xb_gamertag;
+  std::unordered_map<std::string, std::shared_ptr<Account>> by_bb_username;
+
+  void add_locked(std::shared_ptr<Account> a);
+
+  std::shared_ptr<Login> from_dc_nte_credentials_locked(
+      const std::string& serial_number,
+      const std::string& access_key);
+  std::shared_ptr<Login> from_dc_credentials_locked(
+      uint32_t serial_number,
+      const std::string& access_key,
+      const std::string& character_name);
+  std::shared_ptr<Login> from_pc_credentials_locked(
+      uint32_t serial_number,
+      const std::string& access_key,
+      const std::string& character_name);
+  std::shared_ptr<Login> from_gc_credentials_locked(
+      uint32_t serial_number,
+      const std::string& access_key,
+      const std::string* password,
+      const std::string& character_name);
+  std::shared_ptr<Login> from_xb_credentials_locked(
+      const std::string& gamertag,
+      uint64_t user_id,
+      uint64_t account_id);
+  std::shared_ptr<Login> from_bb_credentials_locked(
+      const std::string& username,
+      const std::string* password);
+};

src/AddressTranslator-Stub.hh

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+inline void run_address_translator(const std::string&, const std::string&, const std::string&) {
+  throw std::runtime_error("resource_file is not available; install it and rebuild newserv");
+}
+
+inline std::vector<std::pair<uint32_t, std::string>> diff_dol_files(const std::string&, const std::string&) {
+  throw std::runtime_error("resource_file is not available; install it and rebuild newserv");
+}

src/AddressTranslator.cc

@@ -0,0 +1,528 @@
+#include "AddressTranslator.hh"
+
+#include <array>
+#include <future>
+#include <phosg/Filesystem.hh>
+#include <phosg/Strings.hh>
+#include <resource_file/ExecutableFormats/DOLFile.hh>
+#include <resource_file/ExecutableFormats/XBEFile.hh>
+
+using namespace std;
+
+class AddressTranslator {
+public:
+  enum class ExpandMethod {
+    PPC_TEXT_FORWARD = 0,
+    PPC_TEXT_FORWARD_WITH_BARRIER,
+    PPC_TEXT_BACKWARD,
+    PPC_TEXT_BACKWARD_WITH_BARRIER,
+    PPC_TEXT_BOTH,
+    PPC_TEXT_BOTH_WITH_BARRIER,
+    PPC_TEXT_BOTH_IGNORE_ORIGIN,
+    PPC_DATA_FORWARD,
+    PPC_DATA_BACKWARD,
+    PPC_DATA_BOTH,
+    RAW_FORWARD,
+    RAW_BACKWARD,
+    RAW_BOTH,
+  };
+
+  static const char* name_for_expand_method(ExpandMethod method) {
+    switch (method) {
+      case ExpandMethod::PPC_TEXT_FORWARD:
+        return "PPC_TEXT_FORWARD";
+      case ExpandMethod::PPC_TEXT_FORWARD_WITH_BARRIER:
+        return "PPC_TEXT_FORWARD_WITH_BARRIER";
+      case ExpandMethod::PPC_TEXT_BACKWARD:
+        return "PPC_TEXT_BACKWARD";
+      case ExpandMethod::PPC_TEXT_BACKWARD_WITH_BARRIER:
+        return "PPC_TEXT_BACKWARD_WITH_BARRIER";
+      case ExpandMethod::PPC_TEXT_BOTH:
+        return "PPC_TEXT_BOTH";
+      case ExpandMethod::PPC_TEXT_BOTH_WITH_BARRIER:
+        return "PPC_TEXT_BOTH_WITH_BARRIER";
+      case ExpandMethod::PPC_TEXT_BOTH_IGNORE_ORIGIN:
+        return "PPC_TEXT_BOTH_IGNORE_ORIGIN";
+      case ExpandMethod::PPC_DATA_FORWARD:
+        return "PPC_DATA_FORWARD";
+      case ExpandMethod::PPC_DATA_BACKWARD:
+        return "PPC_DATA_BACKWARD";
+      case ExpandMethod::PPC_DATA_BOTH:
+        return "PPC_DATA_BOTH";
+      case ExpandMethod::RAW_FORWARD:
+        return "RAW_FORWARD";
+      case ExpandMethod::RAW_BACKWARD:
+        return "RAW_BACKWARD";
+      case ExpandMethod::RAW_BOTH:
+        return "RAW_BOTH";
+      default:
+        throw logic_error("invalid expand method");
+    }
+  }
+
+  static bool is_ppc_expand_method(ExpandMethod method) {
+    switch (method) {
+      case ExpandMethod::PPC_TEXT_FORWARD:
+      case ExpandMethod::PPC_TEXT_FORWARD_WITH_BARRIER:
+      case ExpandMethod::PPC_TEXT_BACKWARD:
+      case ExpandMethod::PPC_TEXT_BACKWARD_WITH_BARRIER:
+      case ExpandMethod::PPC_TEXT_BOTH:
+      case ExpandMethod::PPC_TEXT_BOTH_WITH_BARRIER:
+      case ExpandMethod::PPC_TEXT_BOTH_IGNORE_ORIGIN:
+      case ExpandMethod::PPC_DATA_FORWARD:
+      case ExpandMethod::PPC_DATA_BACKWARD:
+      case ExpandMethod::PPC_DATA_BOTH:
+        return true;
+      case ExpandMethod::RAW_FORWARD:
+      case ExpandMethod::RAW_BACKWARD:
+      case ExpandMethod::RAW_BOTH:
+        return false;
+      default:
+        throw logic_error("invalid expand method");
+    }
+  }
+
+  static bool is_ppc_data_expand_method(ExpandMethod method) {
+    switch (method) {
+      case ExpandMethod::PPC_DATA_FORWARD:
+      case ExpandMethod::PPC_DATA_BACKWARD:
+      case ExpandMethod::PPC_DATA_BOTH:
+        return true;
+      case ExpandMethod::PPC_TEXT_FORWARD:
+      case ExpandMethod::PPC_TEXT_FORWARD_WITH_BARRIER:
+      case ExpandMethod::PPC_TEXT_BACKWARD:
+      case ExpandMethod::PPC_TEXT_BACKWARD_WITH_BARRIER:
+      case ExpandMethod::PPC_TEXT_BOTH:
+      case ExpandMethod::PPC_TEXT_BOTH_WITH_BARRIER:
+      case ExpandMethod::PPC_TEXT_BOTH_IGNORE_ORIGIN:
+      case ExpandMethod::RAW_FORWARD:
+      case ExpandMethod::RAW_BACKWARD:
+      case ExpandMethod::RAW_BOTH:
+        return false;
+      default:
+        throw logic_error("invalid expand method");
+    }
+  }
+
+  AddressTranslator(const string& directory)
+      : log("[addr-trans] "),
+        directory(directory),
+        enable_ppc(false) {
+    while (phosg::ends_with(this->directory, "/")) {
+      this->directory.pop_back();
+    }
+    for (const auto& filename : phosg::list_directory(this->directory)) {
+      if (phosg::ends_with(filename, ".dol")) {
+        string name = filename.substr(0, filename.size() - 4);
+        string path = directory + "/" + filename;
+        ResourceDASM::DOLFile dol(path.c_str());
+        auto mem = make_shared<ResourceDASM::MemoryContext>();
+        dol.load_into(mem);
+        this->mems.emplace(name, mem);
+        this->enable_ppc = true;
+        this->log.info("Loaded %s", name.c_str());
+      } else if (phosg::ends_with(filename, ".xbe")) {
+        string name = filename.substr(0, filename.size() - 4);
+        string path = directory + "/" + filename;
+        ResourceDASM::XBEFile xbe(path.c_str());
+        auto mem = make_shared<ResourceDASM::MemoryContext>();
+        xbe.load_into(mem);
+        this->mems.emplace(name, mem);
+        this->log.info("Loaded %s", name.c_str());
+      } else if (phosg::ends_with(filename, ".bin")) {
+        string name = filename.substr(0, filename.size() - 4);
+        string path = directory + "/" + filename;
+        string data = phosg::load_file(path);
+        auto mem = make_shared<ResourceDASM::MemoryContext>();
+        mem->allocate_at(0x8C010000, data.size());
+        mem->memcpy(0x8C010000, data.data(), data.size());
+        this->mems.emplace(name, mem);
+        this->log.info("Loaded %s", name.c_str());
+      }
+    }
+  }
+  ~AddressTranslator() = default;
+
+  const string& get_source_filename() const {
+    return this->src_filename;
+  }
+  void set_source_file(const string& filename) {
+    this->src_filename = filename;
+    this->src_mem = this->mems.at(this->src_filename);
+  }
+
+  void find_ppc_rtoc_global_regs() const {
+    for (const auto& it : this->mems) {
+      bool r2_high_found = false;
+      bool r2_low_found = false;
+      bool r13_high_found = false;
+      bool r13_low_found = false;
+      uint32_t r2 = 0;
+      uint32_t r13 = 0;
+      for (const auto& block : it.second->allocated_blocks()) {
+        phosg::StringReader r = it.second->reader(block.first, block.second);
+        while (!r.eof() && r.where()) {
+          uint32_t opcode = r.get_u32b();
+          if ((opcode & 0xFFFF0000) == 0x3DA00000) {
+            if (r13_high_found) {
+              throw runtime_error("multiple values for r13_high");
+            }
+            r13_high_found = true;
+            r13 |= (opcode << 16);
+          } else if ((opcode & 0xFFFF0000) == 0x3C400000) {
+            if (r2_high_found) {
+              throw runtime_error("multiple values for r2_high");
+            }
+            r2_high_found = true;
+            r2 |= (opcode << 16);
+          } else if ((opcode & 0xFFFF0000) == 0x61AD0000) {
+            if (r13_low_found) {
+              throw runtime_error("multiple values for r13_low");
+            }
+            r13_low_found = true;
+            r13 |= (opcode & 0xFFFF);
+          } else if ((opcode & 0xFFFF0000) == 0x60420000) {
+            if (r2_low_found) {
+              throw runtime_error("multiple values for r2_low");
+            }
+            r2_low_found = true;
+            r2 |= (opcode & 0xFFFF);
+          }
+        }
+      }
+      if (r2_low_found && r2_high_found) {
+        fprintf(stderr, "(%s) r2 = %08" PRIX32 "\n", it.first.c_str(), r2);
+      } else {
+        fprintf(stderr, "(%s) r2 = __MISSING__\n", it.first.c_str());
+      }
+      if (r13_low_found && r13_high_found) {
+        fprintf(stderr, "(%s) r13 = %08" PRIX32 "\n", it.first.c_str(), r13);
+      } else {
+        fprintf(stderr, "(%s) r13 = __MISSING__\n", it.first.c_str());
+      }
+    }
+  }
+
+  uint32_t find_match(
+      shared_ptr<const ResourceDASM::MemoryContext> dest_mem,
+      uint32_t src_addr,
+      uint32_t src_size,
+      ExpandMethod expand_method) const {
+    bool is_ppc = this->is_ppc_expand_method(expand_method);
+    bool is_ppc_data = this->is_ppc_data_expand_method(expand_method);
+    if (!this->src_mem) {
+      throw runtime_error("no source file selected");
+    }
+
+    if (src_size == 0) {
+      src_size = is_ppc ? 4 : 1;
+    }
+
+    pair<uint32_t, uint32_t> src_section = make_pair(0, 0);
+    for (const auto& sec : this->src_mem->allocated_blocks()) {
+      if (src_addr >= sec.first && src_addr + src_size <= sec.first + sec.second) {
+        src_section = sec;
+        break;
+      }
+    }
+    if (!src_section.second) {
+      throw runtime_error("source address not within any section");
+    }
+
+    const char* method_token = this->name_for_expand_method(expand_method);
+
+    size_t src_offset = src_addr - src_section.first;
+    size_t src_bytes_available_before = src_offset;
+    size_t src_bytes_available_after = src_section.second - src_offset - 4;
+    this->log.info("(find_match/%s) Source offset = %08zX with %zX/%zX bytes available before/after",
+        method_token, src_offset, src_bytes_available_before, src_bytes_available_after);
+
+    size_t match_bytes_before = 0;
+    size_t match_bytes_after = 0;
+    while (match_bytes_before + match_bytes_after + 4 < 0x100) {
+      size_t num_matches = 0;
+      size_t last_match_address = 0;
+      size_t match_length = match_bytes_before + match_bytes_after + 4;
+      phosg::StringReader src_r = this->src_mem->reader(src_section.first + src_offset - match_bytes_before, match_length);
+      for (const auto& dest_section : dest_mem->allocated_blocks()) {
+        for (size_t dest_match_offset = 0;
+             dest_match_offset + match_length < dest_section.second;
+             dest_match_offset += (is_ppc ? 4 : 1)) {
+          src_r.go(0);
+          phosg::StringReader dest_r = dest_mem->reader(dest_section.first + dest_match_offset, match_length);
+          size_t z;
+          if (is_ppc) {
+            for (z = 0; z < match_length; z += 4) {
+              if ((expand_method == ExpandMethod::PPC_TEXT_BOTH_IGNORE_ORIGIN) && (z == match_bytes_before)) {
+                src_r.skip(4);
+                dest_r.skip(4);
+              } else if (!is_ppc_data) {
+                uint32_t src_opcode = src_r.get_u32b();
+                uint32_t dest_opcode = dest_r.get_u32b();
+                uint32_t src_class = src_opcode & 0xFC000000;
+                if (src_class != (dest_opcode & 0xFC000000)) {
+                  break;
+                }
+                if (src_class == 0x48000000) {
+                  // b +-offset
+                  src_opcode &= 0xFC000003;
+                  dest_opcode &= 0xFC000003;
+                } else if (((src_opcode & 0xAC1F0000) == 0x800D0000) || ((src_opcode & 0xAC1F0000) == 0x80020000)) {
+                  // lwz/lfs rXX/fXX, [r2/r13 +- offset] OR stw/stfs [r2/r13 +- offset], rXX/fXX
+                  src_opcode &= 0xFFFF0000;
+                  dest_opcode &= 0xFFFF0000;
+                }
+                if (src_opcode != dest_opcode) {
+                  break;
+                }
+              } else {
+                uint32_t src_data = src_r.get_u32b();
+                uint32_t dest_data = dest_r.get_u32b();
+                if ((src_data & 0xFE000000) == 0x80000000) {
+                  src_data &= 0xFE000003;
+                }
+                if ((dest_data & 0xFE000000) == 0x80000000) {
+                  dest_data &= 0xFE000003;
+                }
+                if (src_data != dest_data) {
+                  break;
+                }
+              }
+            }
+          } else {
+            for (z = 0; z < match_length; z++) {
+              uint8_t src_data = src_r.get_u8();
+              uint8_t dest_data = dest_r.get_u8();
+              if (src_data != dest_data) {
+                break;
+              }
+            }
+          }
+          if (z == match_length) {
+            num_matches++;
+            last_match_address = dest_section.first + dest_match_offset + match_bytes_before;
+          }
+        }
+      }
+      this->log.info("(find_match/%s) For match length %zX, %zu matches found", method_token, match_length, num_matches);
+      if (num_matches == 1) {
+        return last_match_address;
+      } else if (num_matches == 0) {
+        throw runtime_error("did not find exactly one match");
+      }
+      bool can_expand_backward = false;
+      bool can_expand_forward = false;
+      switch (expand_method) {
+        case ExpandMethod::PPC_TEXT_BACKWARD_WITH_BARRIER:
+          can_expand_backward = (src_r.pget_u32b(0) != 0x4E800020) &&
+              (src_bytes_available_before >= match_bytes_before + 4);
+          break;
+        case ExpandMethod::PPC_TEXT_BACKWARD:
+        case ExpandMethod::PPC_DATA_BACKWARD:
+          can_expand_backward = (src_bytes_available_before >= match_bytes_before + 4);
+          break;
+        case ExpandMethod::PPC_TEXT_FORWARD_WITH_BARRIER:
+          can_expand_forward = (src_r.pget_u32b(src_r.size() - 4) != 0x4E800020) &&
+              (src_bytes_available_after >= match_bytes_after + 4);
+          break;
+        case ExpandMethod::PPC_TEXT_FORWARD:
+        case ExpandMethod::PPC_DATA_FORWARD:
+          can_expand_forward = (src_bytes_available_after >= match_bytes_after + 4);
+          break;
+        case ExpandMethod::PPC_TEXT_BOTH_WITH_BARRIER:
+        case ExpandMethod::PPC_TEXT_BOTH_IGNORE_ORIGIN:
+          can_expand_backward = (src_r.pget_u32b(0) != 0x4E800020) &&
+              (src_bytes_available_before >= match_bytes_before + 4);
+          can_expand_forward = (src_r.pget_u32b(src_r.size() - 4) != 0x4E800020) &&
+              (src_bytes_available_after >= match_bytes_after + 4);
+          break;
+        case ExpandMethod::PPC_TEXT_BOTH:
+        case ExpandMethod::PPC_DATA_BOTH:
+          can_expand_backward = (src_bytes_available_before >= match_bytes_before + 4);
+          can_expand_forward = (src_bytes_available_after >= match_bytes_after + 4);
+          break;
+        case ExpandMethod::RAW_BACKWARD:
+          can_expand_backward = (src_bytes_available_before > match_bytes_before);
+          break;
+        case ExpandMethod::RAW_FORWARD:
+          can_expand_forward = (src_bytes_available_after > match_bytes_after);
+          break;
+        case ExpandMethod::RAW_BOTH:
+          can_expand_backward = (src_bytes_available_before > match_bytes_before);
+          can_expand_forward = (src_bytes_available_after > match_bytes_after);
+          break;
+        default:
+          throw logic_error("invalid expand method");
+      }
+      if (!can_expand_backward && !can_expand_forward) {
+        throw runtime_error("no further expansion is allowed");
+      }
+      if (can_expand_backward) {
+        match_bytes_before += (is_ppc ? 4 : 1);
+      }
+      if (can_expand_forward) {
+        match_bytes_after += (is_ppc ? 4 : 1);
+      }
+    }
+    throw runtime_error("scan field too long; too many matches");
+  }
+
+  void find_all_matches(uint32_t src_addr, uint32_t src_size) const {
+    if (!this->src_mem) {
+      throw runtime_error("no source file selected");
+    }
+
+    map<string, uint32_t> results;
+    for (const auto& it : this->mems) {
+      if (it.second == this->src_mem) {
+        log.info("(%s) %08" PRIX32 " (from source)", it.first.c_str(), src_addr);
+        results.emplace(it.first, src_addr);
+
+      } else {
+        vector<future<uint32_t>> futures;
+        static const vector<ExpandMethod> ppc_methods = {
+            ExpandMethod::PPC_TEXT_FORWARD,
+            ExpandMethod::PPC_TEXT_FORWARD_WITH_BARRIER,
+            ExpandMethod::PPC_TEXT_BACKWARD,
+            ExpandMethod::PPC_TEXT_BACKWARD_WITH_BARRIER,
+            ExpandMethod::PPC_TEXT_BOTH,
+            ExpandMethod::PPC_TEXT_BOTH_WITH_BARRIER,
+            ExpandMethod::PPC_TEXT_BOTH_IGNORE_ORIGIN,
+            ExpandMethod::PPC_DATA_FORWARD,
+            ExpandMethod::PPC_DATA_BACKWARD,
+            ExpandMethod::PPC_DATA_BOTH,
+        };
+        static const vector<ExpandMethod> raw_methods = {
+            ExpandMethod::RAW_FORWARD,
+            ExpandMethod::RAW_BACKWARD,
+            ExpandMethod::RAW_BOTH,
+        };
+        const auto& methods = this->enable_ppc ? ppc_methods : raw_methods;
+        for (size_t z = 0; z < methods.size(); z++) {
+          futures.emplace_back(async(&AddressTranslator::find_match, this, it.second, src_addr, src_size, methods[z]));
+        }
+
+        unordered_set<uint32_t> match_addrs;
+        for (size_t z = 0; z < futures.size(); z++) {
+          const char* method_name = this->name_for_expand_method(methods[z]);
+          try {
+            uint32_t ret = futures[z].get();
+            log.info("(%s) (%s) %08" PRIX32, it.first.c_str(), method_name, ret);
+            match_addrs.emplace(ret);
+          } catch (const exception& e) {
+            log.error("(%s) (%s) failed: %s", it.first.c_str(), method_name, e.what());
+          }
+        }
+
+        if (match_addrs.empty()) {
+          log.error("(%s) no match found", it.first.c_str());
+        } else if (match_addrs.size() > 1) {
+          log.error("(%s) different matches found by different methods", it.first.c_str());
+        } else {
+          results.emplace(it.first, *match_addrs.begin());
+        }
+      }
+    }
+    for (const auto& it : results) {
+      fprintf(stdout, "%s => %08" PRIX32 "\n", it.first.c_str(), it.second);
+    }
+  }
+
+  void handle_command(const string& command) {
+    auto tokens = phosg::split(command, ' ');
+    if (tokens.empty()) {
+      throw runtime_error("no command given");
+    }
+    phosg::strip_trailing_whitespace(tokens[tokens.size() - 1]);
+
+    if (tokens[0] == "use") {
+      this->set_source_file(tokens.at(1));
+    } else if (tokens[0] == "match") {
+      this->find_all_matches(
+          stoul(tokens.at(1), nullptr, 16),
+          tokens.size() >= 3 ? stoul(tokens[2], nullptr, 16) : 0);
+    } else if (tokens[0] == "find-ppc-globals") {
+      this->find_ppc_rtoc_global_regs();
+    } else if (!tokens[0].empty()) {
+      throw runtime_error("unknown command");
+    }
+  }
+
+  void run_shell() {
+    while (!feof(stdin)) {
+      if (!this->src_filename.empty()) {
+        fprintf(stdout, "addr-trans:%s/%s> ", this->directory.c_str(), this->src_filename.c_str());
+      } else {
+        fprintf(stdout, "addr-trans:%s> ", this->directory.c_str());
+      }
+      fflush(stdout);
+
+      string command = phosg::fgets(stdin);
+      try {
+        this->handle_command(command);
+      } catch (const exception& e) {
+        this->log.error("Failed: %s", e.what());
+      }
+    }
+    fputc('\n', stdout);
+  }
+
+private:
+  phosg::PrefixedLogger log;
+  string directory;
+  unordered_map<string, shared_ptr<const ResourceDASM::MemoryContext>> mems;
+  string src_filename;
+  shared_ptr<const ResourceDASM::MemoryContext> src_mem;
+  bool enable_ppc;
+};
+
+void run_address_translator(const std::string& directory, const std::string& use_filename, const std::string& command) {
+  AddressTranslator trans(directory);
+  if (!use_filename.empty()) {
+    trans.set_source_file(use_filename);
+  }
+
+  if (!command.empty()) {
+    trans.handle_command(command);
+  } else {
+    trans.run_shell();
+  }
+}
+
+vector<pair<uint32_t, string>> diff_dol_files(const string& a_filename, const string& b_filename) {
+  ResourceDASM::DOLFile a(a_filename.c_str());
+  ResourceDASM::DOLFile b(b_filename.c_str());
+  auto a_mem = make_shared<ResourceDASM::MemoryContext>();
+  auto b_mem = make_shared<ResourceDASM::MemoryContext>();
+  a.load_into(a_mem);
+  b.load_into(b_mem);
+
+  uint32_t min_addr = 0xFFFFFFFF;
+  uint32_t max_addr = 0x00000000;
+  for (const auto& sec : a.sections) {
+    min_addr = min<uint32_t>(min_addr, sec.address);
+    max_addr = max<uint32_t>(max_addr, sec.address + sec.data.size());
+  }
+  for (const auto& sec : b.sections) {
+    min_addr = min<uint32_t>(min_addr, sec.address);
+    max_addr = max<uint32_t>(max_addr, sec.address + sec.data.size());
+  }
+
+  vector<pair<uint32_t, string>> ret;
+  for (uint32_t addr = min_addr; addr < max_addr; addr += 4) {
+    bool a_exists = a_mem->exists(addr, 4);
+    bool b_exists = b_mem->exists(addr, 4);
+    if (a_exists && b_exists) {
+      string a_value = a_mem->read(addr, 4);
+      string b_value = b_mem->read(addr, 4);
+      if (a_value != b_value) {
+        if (!ret.empty() && (ret.back().first + ret.back().second.size() == addr)) {
+          ret.back().second += b_value;
+        } else {
+          ret.emplace_back(make_pair(addr, b_value));
+        }
+      }
+    }
+  }
+  return ret;
+}

src/AddressTranslator.hh

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+void run_address_translator(const std::string& directory, const std::string& use_filename, const std::string& command);
+std::vector<std::pair<uint32_t, std::string>> diff_dol_files(const std::string& a_filename, const std::string& b_filename);

src/BMLArchive.cc

@@ -5,40 +5,47 @@
 #include <stdexcept>
 
 #include "Text.hh"
+#include "Types.hh"
 
 using namespace std;
 
-template <bool IsBigEndian>
-struct BMLHeader {
-  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
-
+template <bool BE>
+struct BMLHeaderT {
   parray<uint8_t, 0x04> unknown_a1;
-  U32T num_entries;
+  U32T<BE> num_entries;
   parray<uint8_t, 0x38> unknown_a2;
-} __attribute__((packed));
+} __packed__;
 
-template <bool IsBigEndian>
-struct BMLHeaderEntry {
-  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+using BMLHeader = BMLHeaderT<false>;
+using BMLHeaderBE = BMLHeaderT<true>;
+check_struct_size(BMLHeader, 0x40);
+check_struct_size(BMLHeaderBE, 0x40);
 
+template <bool BE>
+struct BMLHeaderEntryT {
   pstring<TextEncoding::ASCII, 0x20> filename;
-  U32T compressed_size;
+  U32T<BE> compressed_size;
   parray<uint8_t, 0x04> unknown_a1;
-  U32T decompressed_size;
-  U32T compressed_gvm_size;
-  U32T decompressed_gvm_size;
+  U32T<BE> decompressed_size;
+  U32T<BE> compressed_gvm_size;
+  U32T<BE> decompressed_gvm_size;
   parray<uint8_t, 0x0C> unknown_a2;
-} __attribute__((packed));
+} __packed__;
 
-template <bool IsBigEndian>
+using BMLHeaderEntry = BMLHeaderEntryT<false>;
+using BMLHeaderEntryBE = BMLHeaderEntryT<true>;
+check_struct_size(BMLHeaderEntry, 0x40);
+check_struct_size(BMLHeaderEntryBE, 0x40);
+
+template <bool BE>
 void BMLArchive::load_t() {
-  StringReader r(*this->data);
+  phosg::StringReader r(*this->data);
 
-  const auto& header = r.get<BMLHeader<IsBigEndian>>();
+  const auto& header = r.get<BMLHeaderT<BE>>();
 
   size_t offset = 0x800;
   while (this->entries.size() < header.num_entries) {
-    const auto& entry = r.get<BMLHeaderEntry<IsBigEndian>>();
+    const auto& entry = r.get<BMLHeaderEntryT<BE>>();
 
     if (offset + entry.compressed_size > this->data->size()) {
       throw runtime_error("BML data entry extends beyond end of data");
@@ -96,10 +103,10 @@ string BMLArchive::get_copy(const string& name) const {
   }
 }
 
-StringReader BMLArchive::get_reader(const string& name) const {
+phosg::StringReader BMLArchive::get_reader(const string& name) const {
   try {
     const auto& entry = this->entries.at(name);
-    return StringReader(this->data->data() + entry.offset, entry.size);
+    return phosg::StringReader(this->data->data() + entry.offset, entry.size);
   } catch (const out_of_range&) {
     throw out_of_range("BML does not contain file: " + name);
   }

src/BMLArchive.hh

@@ -24,10 +24,10 @@ public:
   std::pair<const void*, size_t> get(const std::string& name) const;
   std::pair<const void*, size_t> get_gvm(const std::string& name) const;
   std::string get_copy(const std::string& name) const;
-  StringReader get_reader(const std::string& name) const;
+  phosg::StringReader get_reader(const std::string& name) const;
 
 private:
-  template <bool IsBigEndian>
+  template <bool BE>
   void load_t();
 
   std::shared_ptr<const std::string> data;

src/BattleParamsIndex.cc

@@ -1,83 +1,83 @@
-#include "BattleParamsIndex.hh"
-
-#include <phosg/Filesystem.hh>
-#include <phosg/Strings.hh>
-
-#include "Loggers.hh"
-#include "PSOEncryption.hh"
-#include "StaticGameData.hh"
-
-using namespace std;
-
-void BattleParamsIndex::Table::print(FILE* stream) const {
-  auto print_entry = +[](FILE* stream, const PlayerStats& e) {
-    fprintf(stream,
-        "%5hu %5hu %5hu %5hu %5hu %5hu %5hu %5hu %5" PRIu32 " %5" PRIu32,
-        e.char_stats.atp.load(),
-        e.char_stats.mst.load(),
-        e.char_stats.evp.load(),
-        e.char_stats.hp.load(),
-        e.char_stats.dfp.load(),
-        e.char_stats.ata.load(),
-        e.char_stats.lck.load(),
-        e.unknown_a1.load(),
-        e.experience.load(),
-        e.meseta.load());
-  };
-
-  for (size_t diff = 0; diff < 4; diff++) {
-    fprintf(stream, "%c ZZ   ATP   PSV   EVP    HP   DFP   ATA   LCK   ESP   EXP  DIFF\n",
-        abbreviation_for_difficulty(diff));
-    for (size_t z = 0; z < 0x60; z++) {
-      fprintf(stream, "  %02zX ", z);
-      print_entry(stream, this->stats[diff][z]);
-      fputc('\n', stream);
-    }
-  }
-}
-
-BattleParamsIndex::BattleParamsIndex(
-    shared_ptr<const string> data_on_ep1,
-    shared_ptr<const string> data_on_ep2,
-    shared_ptr<const string> data_on_ep4,
-    shared_ptr<const string> data_off_ep1,
-    shared_ptr<const string> data_off_ep2,
-    shared_ptr<const string> data_off_ep4) {
-  this->files[0][0].data = data_on_ep1;
-  this->files[0][1].data = data_on_ep2;
-  this->files[0][2].data = data_on_ep4;
-  this->files[1][0].data = data_off_ep1;
-  this->files[1][1].data = data_off_ep2;
-  this->files[1][2].data = data_off_ep4;
-
-  for (uint8_t is_solo = 0; is_solo < 2; is_solo++) {
-    for (uint8_t episode = 0; episode < 3; episode++) {
-      auto& file = this->files[is_solo][episode];
-      if (file.data->size() < sizeof(Table)) {
-        throw runtime_error(string_printf(
-            "battle params table size is incorrect (expected %zX bytes, have %zX bytes; is_solo=%hhu, episode=%hhu)",
-            sizeof(Table), file.data->size(), is_solo, episode));
-      }
-      file.table = reinterpret_cast<const Table*>(file.data->data());
-    }
-  }
-}
-
-const BattleParamsIndex::Table& BattleParamsIndex::get_table(bool solo, Episode episode) const {
-  uint8_t ep_index;
-  switch (episode) {
-    case Episode::EP1:
-      ep_index = 0;
-      break;
-    case Episode::EP2:
-      ep_index = 1;
-      break;
-    case Episode::EP4:
-      ep_index = 2;
-      break;
-    default:
-      throw invalid_argument("invalid episode");
-  }
-
-  return *this->files[!!solo][ep_index].table;
-}
+#include "BattleParamsIndex.hh"
+
+#include <phosg/Filesystem.hh>
+#include <phosg/Strings.hh>
+
+#include "Loggers.hh"
+#include "PSOEncryption.hh"
+#include "StaticGameData.hh"
+
+using namespace std;
+
+void BattleParamsIndex::Table::print(FILE* stream) const {
+  auto print_entry = +[](FILE* stream, const PlayerStats& e) {
+    fprintf(stream,
+        "%5hu %5hu %5hu %5hu %5hu %5hu %5hu %5hu %5" PRIu32 " %5" PRIu32,
+        e.char_stats.atp.load(),
+        e.char_stats.mst.load(),
+        e.char_stats.evp.load(),
+        e.char_stats.hp.load(),
+        e.char_stats.dfp.load(),
+        e.char_stats.ata.load(),
+        e.char_stats.lck.load(),
+        e.esp.load(),
+        e.experience.load(),
+        e.meseta.load());
+  };
+
+  for (size_t diff = 0; diff < 4; diff++) {
+    fprintf(stream, "%c ZZ   ATP   PSV   EVP    HP   DFP   ATA   LCK   ESP   EXP  DIFF\n",
+        abbreviation_for_difficulty(diff));
+    for (size_t z = 0; z < 0x60; z++) {
+      fprintf(stream, "  %02zX ", z);
+      print_entry(stream, this->stats[diff][z]);
+      fputc('\n', stream);
+    }
+  }
+}
+
+BattleParamsIndex::BattleParamsIndex(
+    shared_ptr<const string> data_on_ep1,
+    shared_ptr<const string> data_on_ep2,
+    shared_ptr<const string> data_on_ep4,
+    shared_ptr<const string> data_off_ep1,
+    shared_ptr<const string> data_off_ep2,
+    shared_ptr<const string> data_off_ep4) {
+  this->files[0][0].data = data_on_ep1;
+  this->files[0][1].data = data_on_ep2;
+  this->files[0][2].data = data_on_ep4;
+  this->files[1][0].data = data_off_ep1;
+  this->files[1][1].data = data_off_ep2;
+  this->files[1][2].data = data_off_ep4;
+
+  for (uint8_t is_solo = 0; is_solo < 2; is_solo++) {
+    for (uint8_t episode = 0; episode < 3; episode++) {
+      auto& file = this->files[is_solo][episode];
+      if (file.data->size() < sizeof(Table)) {
+        throw runtime_error(phosg::string_printf(
+            "battle params table size is incorrect (expected %zX bytes, have %zX bytes; is_solo=%hhu, episode=%hhu)",
+            sizeof(Table), file.data->size(), is_solo, episode));
+      }
+      file.table = reinterpret_cast<const Table*>(file.data->data());
+    }
+  }
+}
+
+const BattleParamsIndex::Table& BattleParamsIndex::get_table(bool solo, Episode episode) const {
+  uint8_t ep_index;
+  switch (episode) {
+    case Episode::EP1:
+      ep_index = 0;
+      break;
+    case Episode::EP2:
+      ep_index = 1;
+      break;
+    case Episode::EP4:
+      ep_index = 2;
+      break;
+    default:
+      throw invalid_argument("invalid episode");
+  }
+
+  return *this->files[!!solo][ep_index].table;
+}

src/BattleParamsIndex.hh

@@ -1,100 +1,100 @@
-#pragma once
-
-#include <inttypes.h>
-
-#include <array>
-#include <memory>
-#include <phosg/Encoding.hh>
-#include <random>
-#include <string>
-#include <vector>
-
-#include "EnemyType.hh"
-#include "LevelTable.hh"
-#include "StaticGameData.hh"
-#include "Text.hh"
-
-class BattleParamsIndex {
-public:
-  // These files are little-endian, even on PSO GC.
-
-  struct AttackData {
-    /* 00 */ le_int16_t unknown_a1;
-    /* 02 */ le_int16_t atp;
-    /* 04 */ le_int16_t ata_bonus;
-    /* 06 */ le_uint16_t unknown_a4;
-    /* 08 */ le_float distance_x;
-    /* 0C */ le_float angle_x;
-    /* 10 */ le_float distance_y;
-    /* 14 */ le_uint16_t unknown_a8;
-    /* 16 */ le_uint16_t unknown_a9;
-    /* 18 */ le_uint16_t unknown_a10;
-    /* 1A */ le_uint16_t unknown_a11;
-    /* 1C */ le_uint32_t unknown_a12;
-    /* 20 */ le_uint32_t unknown_a13;
-    /* 24 */ le_uint32_t unknown_a14;
-    /* 28 */ le_uint32_t unknown_a15;
-    /* 2C */ le_uint32_t unknown_a16;
-    /* 30 */
-  } __attribute__((packed));
-
-  struct ResistData {
-    /* 00 */ le_int16_t evp_bonus;
-    /* 02 */ le_uint16_t efr;
-    /* 04 */ le_uint16_t eic;
-    /* 06 */ le_uint16_t eth;
-    /* 08 */ le_uint16_t elt;
-    /* 0A */ le_uint16_t edk;
-    /* 0C */ le_uint32_t unknown_a6;
-    /* 10 */ le_uint32_t unknown_a7;
-    /* 14 */ le_uint32_t unknown_a8;
-    /* 18 */ le_uint32_t unknown_a9;
-    /* 1C */ le_int32_t dfp_bonus;
-    /* 20 */
-  } __attribute__((packed));
-
-  struct MovementData {
-    /* 00 */ le_float idle_move_speed;
-    /* 04 */ le_float idle_animation_speed;
-    /* 08 */ le_float move_speed;
-    /* 0C */ le_float animation_speed;
-    /* 10 */ le_float unknown_a1;
-    /* 14 */ le_float unknown_a2;
-    /* 18 */ le_uint32_t unknown_a3;
-    /* 1C */ le_uint32_t unknown_a4;
-    /* 20 */ le_uint32_t unknown_a5;
-    /* 24 */ le_uint32_t unknown_a6;
-    /* 28 */ le_uint32_t unknown_a7;
-    /* 2C */ le_uint32_t unknown_a8;
-    /* 30 */
-  } __attribute__((packed));
-
-  struct Table {
-    /* 0000 */ parray<parray<PlayerStats, 0x60>, 4> stats;
-    /* 3600 */ parray<parray<AttackData, 0x60>, 4> attack_data;
-    /* 7E00 */ parray<parray<ResistData, 0x60>, 4> resist_data;
-    /* AE00 */ parray<parray<MovementData, 0x60>, 4> movement_data;
-    /* F600 */
-
-    void print(FILE* stream) const;
-  } __attribute__((packed));
-
-  BattleParamsIndex(
-      std::shared_ptr<const std::string> data_on_ep1, // BattleParamEntry_on.dat
-      std::shared_ptr<const std::string> data_on_ep2, // BattleParamEntry_lab_on.dat
-      std::shared_ptr<const std::string> data_on_ep4, // BattleParamEntry_ep4_on.dat
-      std::shared_ptr<const std::string> data_off_ep1, // BattleParamEntry.dat
-      std::shared_ptr<const std::string> data_off_ep2, // BattleParamEntry_lab.dat
-      std::shared_ptr<const std::string> data_off_ep4); // BattleParamEntry_ep4.dat
-
-  const Table& get_table(bool solo, Episode episode) const;
-
-private:
-  struct File {
-    std::shared_ptr<const std::string> data;
-    const Table* table;
-  };
-
-  // Indexed as [online/offline][episode]
-  std::array<std::array<File, 3>, 2> files;
-};
+#pragma once
+
+#include <inttypes.h>
+
+#include <array>
+#include <memory>
+#include <phosg/Encoding.hh>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "EnemyType.hh"
+#include "LevelTable.hh"
+#include "StaticGameData.hh"
+#include "Text.hh"
+
+class BattleParamsIndex {
+public:
+  // These files are little-endian, even on PSO GC.
+
+  struct AttackData {
+    /* 00 */ le_int16_t unknown_a1;
+    /* 02 */ le_int16_t atp;
+    /* 04 */ le_int16_t ata_bonus;
+    /* 06 */ le_uint16_t unknown_a4;
+    /* 08 */ le_float distance_x;
+    /* 0C */ le_uint32_t angle_x; // Out of 0x10000 (high 16 bits are unused)
+    /* 10 */ le_float distance_y;
+    /* 14 */ le_uint16_t unknown_a8;
+    /* 16 */ le_uint16_t unknown_a9;
+    /* 18 */ le_uint16_t unknown_a10;
+    /* 1A */ le_uint16_t unknown_a11;
+    /* 1C */ le_uint32_t unknown_a12;
+    /* 20 */ le_uint32_t unknown_a13;
+    /* 24 */ le_uint32_t unknown_a14;
+    /* 28 */ le_uint32_t unknown_a15;
+    /* 2C */ le_uint32_t unknown_a16;
+    /* 30 */
+  } __packed_ws__(AttackData, 0x30);
+
+  struct ResistData {
+    /* 00 */ le_int16_t evp_bonus;
+    /* 02 */ le_uint16_t efr;
+    /* 04 */ le_uint16_t eic;
+    /* 06 */ le_uint16_t eth;
+    /* 08 */ le_uint16_t elt;
+    /* 0A */ le_uint16_t edk;
+    /* 0C */ le_uint32_t unknown_a6;
+    /* 10 */ le_uint32_t unknown_a7;
+    /* 14 */ le_uint32_t unknown_a8;
+    /* 18 */ le_uint32_t unknown_a9;
+    /* 1C */ le_int32_t dfp_bonus;
+    /* 20 */
+  } __packed_ws__(ResistData, 0x20);
+
+  struct MovementData {
+    /* 00 */ le_float idle_move_speed;
+    /* 04 */ le_float idle_animation_speed;
+    /* 08 */ le_float move_speed;
+    /* 0C */ le_float animation_speed;
+    /* 10 */ le_float unknown_a1;
+    /* 14 */ le_float unknown_a2;
+    /* 18 */ le_uint32_t unknown_a3;
+    /* 1C */ le_uint32_t unknown_a4;
+    /* 20 */ le_uint32_t unknown_a5;
+    /* 24 */ le_uint32_t unknown_a6;
+    /* 28 */ le_uint32_t unknown_a7;
+    /* 2C */ le_uint32_t unknown_a8;
+    /* 30 */
+  } __packed_ws__(MovementData, 0x30);
+
+  struct Table {
+    /* 0000 */ parray<parray<PlayerStats, 0x60>, 4> stats;
+    /* 3600 */ parray<parray<AttackData, 0x60>, 4> attack_data;
+    /* 7E00 */ parray<parray<ResistData, 0x60>, 4> resist_data;
+    /* AE00 */ parray<parray<MovementData, 0x60>, 4> movement_data;
+    /* F600 */
+
+    void print(FILE* stream) const;
+  } __packed_ws__(Table, 0xF600);
+
+  BattleParamsIndex(
+      std::shared_ptr<const std::string> data_on_ep1, // BattleParamEntry_on.dat
+      std::shared_ptr<const std::string> data_on_ep2, // BattleParamEntry_lab_on.dat
+      std::shared_ptr<const std::string> data_on_ep4, // BattleParamEntry_ep4_on.dat
+      std::shared_ptr<const std::string> data_off_ep1, // BattleParamEntry.dat
+      std::shared_ptr<const std::string> data_off_ep2, // BattleParamEntry_lab.dat
+      std::shared_ptr<const std::string> data_off_ep4); // BattleParamEntry_ep4.dat
+
+  const Table& get_table(bool solo, Episode episode) const;
+
+private:
+  struct File {
+    std::shared_ptr<const std::string> data;
+    const Table* table;
+  };
+
+  // Indexed as [online/offline][episode]
+  std::array<std::array<File, 3>, 2> files;
+};

src/CatSession.cc

@@ -1,137 +1,187 @@
-#include "CatSession.hh"
-
-#include <arpa/inet.h>
-#include <ctype.h>
-#include <errno.h>
-#include <event2/buffer.h>
-#include <event2/bufferevent.h>
-#include <event2/event.h>
-#include <event2/listener.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <algorithm>
-#include <iostream>
-#include <phosg/Encoding.hh>
-#include <phosg/Filesystem.hh>
-#include <phosg/Network.hh>
-#include <phosg/Random.hh>
-#include <phosg/Strings.hh>
-#include <phosg/Time.hh>
-
-#include "Loggers.hh"
-#include "PSOProtocol.hh"
-#include "ProxyCommands.hh"
-#include "ReceiveCommands.hh"
-#include "ReceiveSubcommands.hh"
-#include "SendCommands.hh"
-
-using namespace std;
-
-CatSession::CatSession(
-    shared_ptr<struct event_base> base,
-    const struct sockaddr_storage& remote,
-    Version version,
-    shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file)
-    : Shell(base),
-      log(string_printf("[CatSession:%s] ", name_for_enum(version)), proxy_server_log.min_level),
-      channel(
-          version,
-          1,
-          CatSession::dispatch_on_channel_input,
-          CatSession::dispatch_on_channel_error,
-          this,
-          "CatSession"),
-      bb_key_file(bb_key_file) {
-  if (remote.ss_family != AF_INET) {
-    throw runtime_error("remote is not AF_INET");
-  }
-
-  string netloc_str = render_sockaddr_storage(remote);
-  this->log.info("Connecting to %s", netloc_str.c_str());
-
-  struct bufferevent* bev = bufferevent_socket_new(
-      this->base.get(), -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS);
-  if (!bev) {
-    throw runtime_error(string_printf("failed to open socket (%d)", EVUTIL_SOCKET_ERROR()));
-  }
-  this->channel.set_bufferevent(bev);
-
-  if (bufferevent_socket_connect(this->channel.bev.get(),
-          reinterpret_cast<const sockaddr*>(&remote), sizeof(struct sockaddr_in)) != 0) {
-    throw runtime_error(string_printf("failed to connect (%d)", EVUTIL_SOCKET_ERROR()));
-  }
-}
-
-void CatSession::dispatch_on_channel_input(
-    Channel& ch, uint16_t command, uint32_t flag, std::string& data) {
-  auto* session = reinterpret_cast<CatSession*>(ch.context_obj);
-  session->on_channel_input(command, flag, data);
-}
-
-void CatSession::on_channel_input(
-    uint16_t command, uint32_t flag, std::string& data) {
-  if (!uses_v4_encryption(this->channel.version)) {
-    if (command == 0x02 || command == 0x17 || command == 0x91 || command == 0x9B) {
-      const auto& cmd = check_size_t<S_ServerInitDefault_DC_PC_V3_02_17_91_9B>(data, 0xFFFF);
-      if (uses_v3_encryption(this->channel.version)) {
-        this->channel.crypt_in = make_shared<PSOV3Encryption>(cmd.server_key);
-        this->channel.crypt_out = make_shared<PSOV3Encryption>(cmd.client_key);
-        this->log.info("Enabled V3 encryption (server key %08" PRIX32 ", client key %08" PRIX32 ")",
-            cmd.server_key.load(), cmd.client_key.load());
-      } else { // PC, DC, or patch server
-        this->channel.crypt_in = make_shared<PSOV2Encryption>(cmd.server_key);
-        this->channel.crypt_out = make_shared<PSOV2Encryption>(cmd.client_key);
-        this->log.info("Enabled V2 encryption (server key %08" PRIX32 ", client key %08" PRIX32 ")",
-            cmd.server_key.load(), cmd.client_key.load());
-      }
-    }
-  } else { // BB
-    if (command == 0x03 || command == 0x9B) {
-      if (!this->bb_key_file) {
-        throw runtime_error("BB encryption requires a key file");
-      }
-      const auto& cmd = check_size_t<S_ServerInitDefault_BB_03_9B>(data, 0xFFFF);
-      this->channel.crypt_in = make_shared<PSOBBEncryption>(*this->bb_key_file, &cmd.server_key[0], sizeof(cmd.server_key));
-      this->channel.crypt_out = make_shared<PSOBBEncryption>(*this->bb_key_file, &cmd.client_key[0], sizeof(cmd.client_key));
-      this->log.info("Enabled BB encryption");
-    }
-  }
-
-  // TODO: Use the iovec form of print_data here instead of
-  // prepend_command_header (which copies the string)
-  string full_cmd = prepend_command_header(
-      this->channel.version, this->channel.crypt_in.get(), command, flag, data);
-  print_data(stdout, full_cmd, 0, nullptr, PrintDataFlags::PRINT_ASCII | PrintDataFlags::OFFSET_16_BITS);
-}
-
-void CatSession::dispatch_on_channel_error(Channel& ch, short events) {
-  auto* session = reinterpret_cast<CatSession*>(ch.context_obj);
-  session->on_channel_error(events);
-}
-
-void CatSession::on_channel_error(short events) {
-  if (events & BEV_EVENT_CONNECTED) {
-    this->log.info("Channel connected");
-  }
-  if (events & BEV_EVENT_ERROR) {
-    int err = EVUTIL_SOCKET_ERROR();
-    this->log.warning("Error %d (%s) in unlinked client stream", err,
-        evutil_socket_error_to_string(err));
-  }
-  if (events & (BEV_EVENT_ERROR | BEV_EVENT_EOF)) {
-    this->log.info("Session endpoint has disconnected");
-    this->channel.disconnect();
-    event_base_loopexit(this->base.get(), nullptr);
-  }
-}
-
-void CatSession::print_prompt() {}
-
-void CatSession::execute_command(const std::string& command) {
-  string full_cmd = parse_data_string(command, nullptr, ParseDataFlags::ALLOW_FILES);
-  send_command_with_header(this->channel, full_cmd.data(), full_cmd.size());
-}
+#include "CatSession.hh"
+
+#include <arpa/inet.h>
+#include <ctype.h>
+#include <errno.h>
+#include <event2/buffer.h>
+#include <event2/bufferevent.h>
+#include <event2/event.h>
+#include <event2/listener.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <iostream>
+#include <phosg/Encoding.hh>
+#include <phosg/Filesystem.hh>
+#include <phosg/Network.hh>
+#include <phosg/Random.hh>
+#include <phosg/Strings.hh>
+#include <phosg/Time.hh>
+
+#include "Loggers.hh"
+#include "PSOProtocol.hh"
+#include "ProxyCommands.hh"
+#include "ReceiveCommands.hh"
+#include "ReceiveSubcommands.hh"
+#include "SendCommands.hh"
+
+using namespace std;
+
+CatSession::exit_shell::exit_shell() : runtime_error("shell exited") {}
+
+CatSession::CatSession(
+    shared_ptr<struct event_base> base,
+    const struct sockaddr_storage& remote,
+    Version version,
+    shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file)
+    : log(phosg::string_printf("[CatSession:%s] ", phosg::name_for_enum(version)), proxy_server_log.min_level),
+      base(base),
+      read_event(event_new(this->base.get(), 0, EV_READ | EV_PERSIST, CatSession::dispatch_read_stdin, this), event_free),
+      channel(version, 1, CatSession::dispatch_on_channel_input, CatSession::dispatch_on_channel_error, this, "CatSession"),
+      bb_key_file(bb_key_file) {
+
+  if (remote.ss_family != AF_INET) {
+    throw runtime_error("remote is not AF_INET");
+  }
+
+  string netloc_str = phosg::render_sockaddr_storage(remote);
+  this->log.info("Connecting to %s", netloc_str.c_str());
+
+  struct bufferevent* bev = bufferevent_socket_new(
+      this->base.get(), -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS);
+  if (!bev) {
+    throw runtime_error(phosg::string_printf("failed to open socket (%d)", EVUTIL_SOCKET_ERROR()));
+  }
+  this->channel.set_bufferevent(bev, 0);
+
+  if (bufferevent_socket_connect(this->channel.bev.get(),
+          reinterpret_cast<const sockaddr*>(&remote), sizeof(struct sockaddr_in)) != 0) {
+    throw runtime_error(phosg::string_printf("failed to connect (%d)", EVUTIL_SOCKET_ERROR()));
+  }
+
+  event_add(this->read_event.get(), nullptr);
+  this->poll.add(0, POLLIN);
+}
+
+void CatSession::execute_command(const std::string& command) {
+  string full_cmd = phosg::parse_data_string(command, nullptr, phosg::ParseDataFlags::ALLOW_FILES);
+  send_command_with_header(this->channel, full_cmd.data(), full_cmd.size());
+}
+
+void CatSession::dispatch_on_channel_input(
+    Channel& ch, uint16_t command, uint32_t flag, std::string& data) {
+  auto* session = reinterpret_cast<CatSession*>(ch.context_obj);
+  session->on_channel_input(command, flag, data);
+}
+
+void CatSession::on_channel_input(
+    uint16_t command, uint32_t flag, std::string& data) {
+  if (!uses_v4_encryption(this->channel.version)) {
+    if (command == 0x02 || command == 0x17 || command == 0x91 || command == 0x9B) {
+      const auto& cmd = check_size_t<S_ServerInitDefault_DC_PC_V3_02_17_91_9B>(data, 0xFFFF);
+      if (uses_v3_encryption(this->channel.version)) {
+        this->channel.crypt_in = make_shared<PSOV3Encryption>(cmd.server_key);
+        this->channel.crypt_out = make_shared<PSOV3Encryption>(cmd.client_key);
+        this->log.info("Enabled V3 encryption (server key %08" PRIX32 ", client key %08" PRIX32 ")",
+            cmd.server_key.load(), cmd.client_key.load());
+      } else { // PC, DC, or patch server
+        this->channel.crypt_in = make_shared<PSOV2Encryption>(cmd.server_key);
+        this->channel.crypt_out = make_shared<PSOV2Encryption>(cmd.client_key);
+        this->log.info("Enabled V2 encryption (server key %08" PRIX32 ", client key %08" PRIX32 ")",
+            cmd.server_key.load(), cmd.client_key.load());
+      }
+    }
+  } else { // BB
+    if (command == 0x03 || command == 0x9B) {
+      if (!this->bb_key_file) {
+        throw runtime_error("BB encryption requires a key file");
+      }
+      const auto& cmd = check_size_t<S_ServerInitDefault_BB_03_9B>(data, 0xFFFF);
+      this->channel.crypt_in = make_shared<PSOBBEncryption>(*this->bb_key_file, &cmd.server_key[0], sizeof(cmd.server_key));
+      this->channel.crypt_out = make_shared<PSOBBEncryption>(*this->bb_key_file, &cmd.client_key[0], sizeof(cmd.client_key));
+      this->log.info("Enabled BB encryption");
+    }
+  }
+
+  // TODO: Use the iovec form of print_data here instead of
+  // prepend_command_header (which copies the string)
+  string full_cmd = prepend_command_header(this->channel.version, this->channel.crypt_in.get(), command, flag, data);
+  phosg::print_data(stdout, full_cmd, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::OFFSET_16_BITS);
+}
+
+void CatSession::dispatch_on_channel_error(Channel& ch, short events) {
+  auto* session = reinterpret_cast<CatSession*>(ch.context_obj);
+  session->on_channel_error(events);
+}
+
+void CatSession::on_channel_error(short events) {
+  if (events & BEV_EVENT_CONNECTED) {
+    this->log.info("Channel connected");
+  }
+  if (events & BEV_EVENT_ERROR) {
+    int err = EVUTIL_SOCKET_ERROR();
+    this->log.warning("Error %d (%s) in unlinked client stream", err,
+        evutil_socket_error_to_string(err));
+  }
+  if (events & (BEV_EVENT_ERROR | BEV_EVENT_EOF)) {
+    this->log.info("Session endpoint has disconnected");
+    this->channel.disconnect();
+    event_base_loopexit(this->base.get(), nullptr);
+  }
+}
+
+void CatSession::dispatch_read_stdin(evutil_socket_t, short, void* ctx) {
+  reinterpret_cast<CatSession*>(ctx)->read_stdin();
+}
+
+void CatSession::read_stdin() {
+  bool any_command_read = false;
+  for (;;) {
+    auto poll_result = this->poll.poll();
+    short fd_events = 0;
+    try {
+      fd_events = poll_result.at(0);
+    } catch (const out_of_range&) {
+    }
+
+    if (!(fd_events & POLLIN)) {
+      break;
+    }
+
+    string command(2048, '\0');
+    if (!fgets(command.data(), command.size(), stdin)) {
+      if (!any_command_read) {
+        // ctrl+d probably; we should exit
+        fputc('\n', stderr);
+        event_base_loopexit(this->base.get(), nullptr);
+        return;
+      } else {
+        break; // probably not EOF; just no more commands for now
+      }
+    }
+
+    // trim the extra data off the string
+    size_t len = strlen(command.c_str());
+    if (len == 0) {
+      break;
+    }
+    if (command[len - 1] == '\n') {
+      len--;
+    }
+    command.resize(len);
+    any_command_read = true;
+
+    try {
+      execute_command(command);
+    } catch (const exit_shell&) {
+      event_base_loopexit(this->base.get(), nullptr);
+      return;
+    } catch (const exception& e) {
+      fprintf(stderr, "FAILED: %s\n", e.what());
+    }
+  }
+}

src/CatSession.hh

@@ -1,41 +1,54 @@
-#pragma once
-
-#include <event2/event.h>
-
-#include <functional>
-#include <map>
-#include <memory>
-#include <phosg/Filesystem.hh>
-#include <string>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-
-#include "PSOEncryption.hh"
-#include "PSOProtocol.hh"
-#include "ServerState.hh"
-#include "Shell.hh"
-
-class CatSession : public Shell {
-public:
-  CatSession(
-      std::shared_ptr<struct event_base> base,
-      const struct sockaddr_storage& remote,
-      Version version,
-      std::shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file);
-  virtual ~CatSession() = default;
-
-protected:
-  PrefixedLogger log;
-  Channel channel;
-  std::shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file;
-
-  virtual void print_prompt();
-  virtual void execute_command(const std::string& command);
-
-  static void dispatch_on_channel_input(
-      Channel& ch, uint16_t command, uint32_t flag, std::string& msg);
-  static void dispatch_on_channel_error(Channel& ch, short events);
-  void on_channel_input(uint16_t command, uint32_t flag, std::string& msg);
-  void on_channel_error(short events);
-};
+#pragma once
+
+#include <event2/event.h>
+
+#include <functional>
+#include <map>
+#include <memory>
+#include <phosg/Filesystem.hh>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "PSOEncryption.hh"
+#include "PSOProtocol.hh"
+#include "ServerState.hh"
+
+class CatSession {
+public:
+  CatSession(
+      std::shared_ptr<struct event_base> base,
+      const struct sockaddr_storage& remote,
+      Version version,
+      std::shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file);
+  CatSession(const CatSession&) = delete;
+  CatSession(CatSession&&) = delete;
+  CatSession& operator=(const CatSession&) = delete;
+  CatSession& operator=(CatSession&&) = delete;
+  virtual ~CatSession() = default;
+
+protected:
+  phosg::PrefixedLogger log;
+  std::shared_ptr<struct event_base> base;
+  std::unique_ptr<struct event, void (*)(struct event*)> read_event;
+  phosg::Poll poll;
+
+  Channel channel;
+  std::shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file;
+
+  class exit_shell : public std::runtime_error {
+  public:
+    exit_shell();
+    ~exit_shell() = default;
+  };
+
+  virtual void execute_command(const std::string& command);
+
+  static void dispatch_read_stdin(evutil_socket_t fd, short events, void* ctx);
+  static void dispatch_on_channel_input(Channel& ch, uint16_t command, uint32_t flag, std::string& msg);
+  static void dispatch_on_channel_error(Channel& ch, short events);
+  void on_channel_input(uint16_t command, uint32_t flag, std::string& msg);
+  void on_channel_error(short events);
+  void read_stdin();
+};

src/Channel.cc

@@ -1,424 +1,421 @@
-#include "Channel.hh"
-
-#include <errno.h>
-#include <event2/buffer.h>
-#include <event2/bufferevent.h>
-#include <event2/event.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <phosg/Network.hh>
-#include <phosg/Time.hh>
-
-#include "Loggers.hh"
-#include "Version.hh"
-
-using namespace std;
-
-extern bool use_terminal_colors;
-
-static void flush_and_free_bufferevent(struct bufferevent* bev) {
-  bufferevent_flush(bev, EV_READ | EV_WRITE, BEV_FINISHED);
-  bufferevent_free(bev);
-}
-
-Channel::Channel(
-    Version version,
-    uint8_t language,
-    on_command_received_t on_command_received,
-    on_error_t on_error,
-    void* context_obj,
-    const string& name,
-    TerminalFormat terminal_send_color,
-    TerminalFormat terminal_recv_color)
-    : bev(nullptr, flush_and_free_bufferevent),
-      version(version),
-      language(language),
-      name(name),
-      terminal_send_color(terminal_send_color),
-      terminal_recv_color(terminal_recv_color),
-      on_command_received(on_command_received),
-      on_error(on_error),
-      context_obj(context_obj) {
-}
-
-Channel::Channel(
-    struct bufferevent* bev,
-    Version version,
-    uint8_t language,
-    on_command_received_t on_command_received,
-    on_error_t on_error,
-    void* context_obj,
-    const string& name,
-    TerminalFormat terminal_send_color,
-    TerminalFormat terminal_recv_color)
-    : bev(nullptr, flush_and_free_bufferevent),
-      version(version),
-      language(language),
-      name(name),
-      terminal_send_color(terminal_send_color),
-      terminal_recv_color(terminal_recv_color),
-      on_command_received(on_command_received),
-      on_error(on_error),
-      context_obj(context_obj) {
-  this->set_bufferevent(bev);
-}
-
-void Channel::replace_with(
-    Channel&& other,
-    on_command_received_t on_command_received,
-    on_error_t on_error,
-    void* context_obj,
-    const std::string& name) {
-  this->set_bufferevent(other.bev.release());
-  this->local_addr = other.local_addr;
-  this->remote_addr = other.remote_addr;
-  this->is_virtual_connection = other.is_virtual_connection;
-  this->version = other.version;
-  this->language = other.language;
-  this->crypt_in = other.crypt_in;
-  this->crypt_out = other.crypt_out;
-  this->name = name;
-  this->terminal_send_color = other.terminal_send_color;
-  this->terminal_recv_color = other.terminal_recv_color;
-  this->on_command_received = on_command_received;
-  this->on_error = on_error;
-  this->context_obj = context_obj;
-  other.disconnect(); // Clears crypts, addrs, etc.
-}
-
-void Channel::set_bufferevent(struct bufferevent* bev) {
-  this->bev.reset(bev);
-
-  if (this->bev.get()) {
-    int fd = bufferevent_getfd(this->bev.get());
-    if (fd < 0) {
-      this->is_virtual_connection = true;
-      memset(&this->local_addr, 0, sizeof(this->local_addr));
-      memset(&this->remote_addr, 0, sizeof(this->remote_addr));
-    } else {
-      this->is_virtual_connection = false;
-      get_socket_addresses(fd, &this->local_addr, &this->remote_addr);
-    }
-
-    bufferevent_setcb(this->bev.get(),
-        &Channel::dispatch_on_input, nullptr,
-        &Channel::dispatch_on_error, this);
-    bufferevent_enable(this->bev.get(), EV_READ | EV_WRITE);
-
-  } else {
-    this->is_virtual_connection = false;
-    memset(&this->local_addr, 0, sizeof(this->local_addr));
-    memset(&this->remote_addr, 0, sizeof(this->remote_addr));
-  }
-}
-
-void Channel::disconnect() {
-  if (this->bev.get()) {
-    // If the output buffer is not empty, move the bufferevent into the draining
-    // pool instead of disconnecting it, to make sure all the data gets sent.
-    struct evbuffer* out_buffer = bufferevent_get_output(this->bev.get());
-    if (evbuffer_get_length(out_buffer) == 0) {
-      this->bev.reset(); // Destructor flushes and frees the bufferevent
-    } else {
-      // The callbacks will free it when all the data is sent or the client
-      // disconnects
-
-      auto on_output = +[](struct bufferevent* bev, void*) -> void {
-        flush_and_free_bufferevent(bev);
-      };
-
-      auto on_error = +[](struct bufferevent* bev, short events, void*) -> void {
-        if (events & BEV_EVENT_ERROR) {
-          int err = EVUTIL_SOCKET_ERROR();
-          channel_exceptions_log.warning(
-              "Disconnecting channel caused error %d (%s)", err,
-              evutil_socket_error_to_string(err));
-        }
-        if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
-          bufferevent_flush(bev, EV_WRITE, BEV_FINISHED);
-          bufferevent_free(bev);
-        }
-      };
-
-      struct bufferevent* bev = this->bev.release();
-      bufferevent_setcb(bev, nullptr, on_output, on_error, bev);
-      bufferevent_disable(bev, EV_READ);
-    }
-  }
-
-  memset(&this->local_addr, 0, sizeof(this->local_addr));
-  memset(&this->remote_addr, 0, sizeof(this->remote_addr));
-  this->is_virtual_connection = false;
-  this->crypt_in.reset();
-  this->crypt_out.reset();
-}
-
-Channel::Message Channel::recv() {
-  struct evbuffer* buf = bufferevent_get_input(this->bev.get());
-
-  size_t header_size = (this->version == Version::BB_V4) ? 8 : 4;
-  PSOCommandHeader header;
-  if (evbuffer_copyout(buf, &header, header_size) < static_cast<ssize_t>(header_size)) {
-    throw out_of_range("no command available");
-  }
-
-  if (this->crypt_in.get()) {
-    this->crypt_in->decrypt(&header, header_size, false);
-  }
-
-  size_t command_logical_size = header.size(version);
-
-  // If encryption is enabled, BB pads commands to 8-byte boundaries, and this
-  // is not reflected in the size field. This logic does not occur if encryption
-  // is not yet enabled.
-  size_t command_physical_size = (this->crypt_in.get() && (version == Version::BB_V4))
-      ? ((command_logical_size + 7) & ~7)
-      : command_logical_size;
-  if (evbuffer_get_length(buf) < command_physical_size) {
-    throw out_of_range("no command available");
-  }
-
-  // If we get here, then there is a full command in the buffer. Some encryption
-  // algorithms' advancement depends on the decrypted data, so we have to
-  // actually decrypt the header again (with advance=true) to keep them in a
-  // consistent state.
-
-  string header_data(header_size, '\0');
-  if (evbuffer_remove(buf, header_data.data(), header_data.size()) < static_cast<ssize_t>(header_data.size())) {
-    throw logic_error("enough bytes available, but could not remove them");
-  }
-  if (this->crypt_in.get()) {
-    this->crypt_in->decrypt(header_data.data(), header_data.size());
-  }
-
-  string command_data(command_physical_size - header_size, '\0');
-  if (evbuffer_remove(buf, command_data.data(), command_data.size()) < static_cast<ssize_t>(command_data.size())) {
-    throw logic_error("enough bytes available, but could not remove them");
-  }
-
-  if (this->crypt_in.get()) {
-    // Some versions of PSO DC can send commands whose sizes are not a multiple
-    // of 4, but the server is expected to always use a multiple of 4 bytes when
-    // decrypting (the extra cipher bytes are lost). To emulate this behavior,
-    // we have to round up the size for DC commands here.
-    size_t orig_size = command_data.size();
-    command_data.resize((orig_size + 3) & (~3), 0);
-    this->crypt_in->decrypt(command_data.data(), command_data.size());
-    command_data.resize(orig_size);
-  }
-  command_data.resize(command_logical_size - header_size);
-
-  if (command_data_log.should_log(LogLevel::INFO) && (this->terminal_recv_color != TerminalFormat::END)) {
-    if (use_terminal_colors && this->terminal_recv_color != TerminalFormat::NORMAL) {
-      print_color_escape(stderr, this->terminal_recv_color, TerminalFormat::BOLD, TerminalFormat::END);
-    }
-
-    if (version == Version::BB_V4) {
-      command_data_log.info(
-          "Received from %s (version=BB command=%04hX flag=%08" PRIX32 ")",
-          this->name.c_str(),
-          header.command(this->version),
-          header.flag(this->version));
-    } else {
-      command_data_log.info(
-          "Received from %s (version=%s command=%02hX flag=%02" PRIX32 ")",
-          this->name.c_str(),
-          name_for_enum(this->version),
-          header.command(this->version),
-          header.flag(this->version));
-    }
-
-    vector<struct iovec> iovs;
-    iovs.emplace_back(iovec{.iov_base = header_data.data(), .iov_len = header_data.size()});
-    iovs.emplace_back(iovec{.iov_base = command_data.data(), .iov_len = command_data.size()});
-    print_data(stderr, iovs, 0, nullptr, PrintDataFlags::PRINT_ASCII | PrintDataFlags::DISABLE_COLOR | PrintDataFlags::OFFSET_16_BITS);
-
-    if (use_terminal_colors && this->terminal_recv_color != TerminalFormat::NORMAL) {
-      print_color_escape(stderr, TerminalFormat::NORMAL, TerminalFormat::END);
-    }
-  }
-
-  return {
-      .command = header.command(this->version),
-      .flag = header.flag(this->version),
-      .data = std::move(command_data),
-  };
-}
-
-void Channel::send(uint16_t cmd, uint32_t flag, bool silent) {
-  this->send(cmd, flag, nullptr, 0, silent);
-}
-
-void Channel::send(uint16_t cmd, uint32_t flag, const std::vector<std::pair<const void*, size_t>> blocks, bool silent) {
-  if (!this->connected()) {
-    channel_exceptions_log.warning("Attempted to send command on closed channel; dropping data");
-    return;
-  }
-
-  size_t size = 0;
-  for (const auto& b : blocks) {
-    size += b.second;
-  }
-
-  string send_data;
-  size_t logical_size;
-  size_t send_data_size = 0;
-  switch (this->version) {
-    case Version::DC_NTE:
-    case Version::DC_V1_11_2000_PROTOTYPE:
-    case Version::DC_V1:
-    case Version::DC_V2:
-    case Version::GC_NTE:
-    case Version::GC_V3:
-    case Version::GC_EP3_NTE:
-    case Version::GC_EP3:
-    case Version::XB_V3: {
-      PSOCommandHeaderDCV3 header;
-      if (this->crypt_out.get() &&
-          (this->version != Version::DC_NTE) &&
-          (this->version != Version::DC_V1_11_2000_PROTOTYPE) &&
-          (this->version != Version::DC_V1)) {
-        send_data_size = (sizeof(header) + size + 3) & ~3;
-      } else {
-        send_data_size = (sizeof(header) + size);
-      }
-      logical_size = send_data_size;
-      header.command = cmd;
-      header.flag = flag;
-      header.size = send_data_size;
-      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
-      break;
-    }
-    case Version::PC_PATCH:
-    case Version::BB_PATCH:
-    case Version::PC_NTE:
-    case Version::PC_V2: {
-      PSOCommandHeaderPC header;
-      if (this->crypt_out.get()) {
-        send_data_size = (sizeof(header) + size + 3) & ~3;
-      } else {
-        send_data_size = (sizeof(header) + size);
-      }
-      logical_size = send_data_size;
-      header.size = send_data_size;
-      header.command = cmd;
-      header.flag = flag;
-      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
-      break;
-    }
-    case Version::BB_V4: {
-      // BB has an annoying behavior here: command lengths must be multiples of
-      // 4, but the actual data length must be a multiple of 8. If the size
-      // field is not divisible by 8, 4 extra bytes are sent anyway. This
-      // behavior only applies when encryption is enabled - any commands sent
-      // before encryption is enabled have no size restrictions (except they
-      // must include a full header and must fit in the client's receive
-      // buffer), and no implicit extra bytes are sent.
-      PSOCommandHeaderBB header;
-      if (this->crypt_out.get()) {
-        send_data_size = (sizeof(header) + size + 7) & ~7;
-      } else {
-        send_data_size = (sizeof(header) + size);
-      }
-      logical_size = (sizeof(header) + size + 3) & ~3;
-      header.size = logical_size;
-      header.command = cmd;
-      header.flag = flag;
-      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
-      break;
-    }
-
-    default:
-      throw logic_error("unimplemented game version in send_command");
-  }
-
-  // All versions of PSO I've seen (so far) have a receive buffer 0x7C00
-  // bytes in size
-  if (send_data_size > 0x7C00) {
-    throw runtime_error("outbound command too large");
-  }
-
-  send_data.reserve(send_data_size);
-  for (const auto& b : blocks) {
-    send_data.append(reinterpret_cast<const char*>(b.first), b.second);
-  }
-  send_data.resize(send_data_size, '\0');
-
-  if (!silent && (command_data_log.should_log(LogLevel::INFO)) && (this->terminal_send_color != TerminalFormat::END)) {
-    if (use_terminal_colors && this->terminal_send_color != TerminalFormat::NORMAL) {
-      print_color_escape(stderr, TerminalFormat::FG_YELLOW, TerminalFormat::BOLD, TerminalFormat::END);
-    }
-    if (version == Version::BB_V4) {
-      command_data_log.info("Sending to %s (version=BB command=%04hX flag=%08" PRIX32 ")",
-          this->name.c_str(), cmd, flag);
-    } else {
-      command_data_log.info("Sending to %s (version=%s command=%02hX flag=%02" PRIX32 ")",
-          this->name.c_str(), name_for_enum(version), cmd, flag);
-    }
-    print_data(stderr, send_data.data(), logical_size, 0, nullptr, PrintDataFlags::PRINT_ASCII | PrintDataFlags::DISABLE_COLOR | PrintDataFlags::OFFSET_16_BITS);
-    if (use_terminal_colors && this->terminal_send_color != TerminalFormat::NORMAL) {
-      print_color_escape(stderr, TerminalFormat::NORMAL, TerminalFormat::END);
-    }
-  }
-
-  if (this->crypt_out.get()) {
-    this->crypt_out->encrypt(send_data.data(), send_data.size());
-  }
-
-  struct evbuffer* buf = bufferevent_get_output(this->bev.get());
-  evbuffer_add(buf, send_data.data(), send_data.size());
-}
-
-void Channel::send(uint16_t cmd, uint32_t flag, const void* data, size_t size, bool silent) {
-  this->send(cmd, flag, {make_pair(data, size)}, silent);
-}
-
-void Channel::send(uint16_t cmd, uint32_t flag, const string& data, bool silent) {
-  this->send(cmd, flag, data.data(), data.size(), silent);
-}
-
-void Channel::send(const void* data, size_t size, bool silent) {
-  size_t header_size = (this->version == Version::BB_V4) ? 8 : 4;
-  const auto* header = reinterpret_cast<const PSOCommandHeader*>(data);
-  this->send(
-      header->command(this->version),
-      header->flag(this->version),
-      reinterpret_cast<const uint8_t*>(data) + header_size,
-      size - header_size,
-      silent);
-}
-
-void Channel::send(const string& data, bool silent) {
-  return this->send(data.data(), data.size(), silent);
-}
-
-void Channel::dispatch_on_input(struct bufferevent*, void* ctx) {
-  Channel* ch = reinterpret_cast<Channel*>(ctx);
-  // The client can be disconnected during on_command_received, so we have to
-  // make sure ch->bev is valid every time before calling recv()
-  while (ch->bev.get()) {
-    Message msg;
-    try {
-      msg = ch->recv();
-    } catch (const out_of_range&) {
-      break;
-    } catch (const exception& e) {
-      channel_exceptions_log.warning("Error receiving on channel: %s", e.what());
-      ch->on_error(*ch, BEV_EVENT_ERROR);
-      break;
-    }
-    if (ch->on_command_received) {
-      ch->on_command_received(*ch, msg.command, msg.flag, msg.data);
-    }
-  }
-}
-
-void Channel::dispatch_on_error(struct bufferevent*, short events, void* ctx) {
-  Channel* ch = reinterpret_cast<Channel*>(ctx);
-  if (ch->on_error) {
-    ch->on_error(*ch, events);
-  } else {
-    ch->disconnect();
-  }
-}
+#include "Channel.hh"
+
+#include <errno.h>
+#include <event2/buffer.h>
+#include <event2/bufferevent.h>
+#include <event2/event.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <phosg/Network.hh>
+#include <phosg/Time.hh>
+
+#include "Loggers.hh"
+#include "Version.hh"
+
+using namespace std;
+
+extern bool use_terminal_colors;
+
+static void flush_and_free_bufferevent(struct bufferevent* bev) {
+  bufferevent_flush(bev, EV_READ | EV_WRITE, BEV_FINISHED);
+  bufferevent_free(bev);
+}
+
+Channel::Channel(
+    Version version,
+    uint8_t language,
+    on_command_received_t on_command_received,
+    on_error_t on_error,
+    void* context_obj,
+    const string& name,
+    phosg::TerminalFormat terminal_send_color,
+    phosg::TerminalFormat terminal_recv_color)
+    : bev(nullptr, flush_and_free_bufferevent),
+      virtual_network_id(0),
+      version(version),
+      language(language),
+      name(name),
+      terminal_send_color(terminal_send_color),
+      terminal_recv_color(terminal_recv_color),
+      on_command_received(on_command_received),
+      on_error(on_error),
+      context_obj(context_obj) {
+}
+
+Channel::Channel(
+    struct bufferevent* bev,
+    uint64_t virtual_network_id,
+    Version version,
+    uint8_t language,
+    on_command_received_t on_command_received,
+    on_error_t on_error,
+    void* context_obj,
+    const string& name,
+    phosg::TerminalFormat terminal_send_color,
+    phosg::TerminalFormat terminal_recv_color)
+    : bev(nullptr, flush_and_free_bufferevent),
+      version(version),
+      language(language),
+      name(name),
+      terminal_send_color(terminal_send_color),
+      terminal_recv_color(terminal_recv_color),
+      on_command_received(on_command_received),
+      on_error(on_error),
+      context_obj(context_obj) {
+  this->set_bufferevent(bev, virtual_network_id);
+}
+
+void Channel::replace_with(
+    Channel&& other,
+    on_command_received_t on_command_received,
+    on_error_t on_error,
+    void* context_obj,
+    const std::string& name) {
+  this->set_bufferevent(other.bev.release(), other.virtual_network_id);
+  this->local_addr = other.local_addr;
+  this->remote_addr = other.remote_addr;
+  this->version = other.version;
+  this->language = other.language;
+  this->crypt_in = other.crypt_in;
+  this->crypt_out = other.crypt_out;
+  this->name = name;
+  this->terminal_send_color = other.terminal_send_color;
+  this->terminal_recv_color = other.terminal_recv_color;
+  this->on_command_received = on_command_received;
+  this->on_error = on_error;
+  this->context_obj = context_obj;
+  other.disconnect(); // Clears crypts, addrs, etc.
+}
+
+void Channel::set_bufferevent(struct bufferevent* bev, uint64_t virtual_network_id) {
+  this->bev.reset(bev);
+  this->virtual_network_id = virtual_network_id;
+
+  if (this->bev.get()) {
+    int fd = bufferevent_getfd(this->bev.get());
+    if (fd < 0) {
+      memset(&this->local_addr, 0, sizeof(this->local_addr));
+      memset(&this->remote_addr, 0, sizeof(this->remote_addr));
+    } else {
+      phosg::get_socket_addresses(fd, &this->local_addr, &this->remote_addr);
+    }
+
+    bufferevent_setcb(this->bev.get(), &Channel::dispatch_on_input, nullptr, &Channel::dispatch_on_error, this);
+    bufferevent_enable(this->bev.get(), EV_READ | EV_WRITE);
+
+  } else {
+    memset(&this->local_addr, 0, sizeof(this->local_addr));
+    memset(&this->remote_addr, 0, sizeof(this->remote_addr));
+  }
+}
+
+void Channel::disconnect() {
+  if (this->bev.get()) {
+    // If the output buffer is not empty, move the bufferevent into the draining
+    // pool instead of disconnecting it, to make sure all the data gets sent.
+    struct evbuffer* out_buffer = bufferevent_get_output(this->bev.get());
+    if (evbuffer_get_length(out_buffer) == 0) {
+      this->bev.reset(); // Destructor flushes and frees the bufferevent
+    } else {
+      // The callbacks will free it when all the data is sent or the client
+      // disconnects
+
+      auto on_output = +[](struct bufferevent* bev, void*) -> void {
+        flush_and_free_bufferevent(bev);
+      };
+
+      auto on_error = +[](struct bufferevent* bev, short events, void*) -> void {
+        if (events & BEV_EVENT_ERROR) {
+          int err = EVUTIL_SOCKET_ERROR();
+          channel_exceptions_log.warning(
+              "Disconnecting channel caused error %d (%s)", err,
+              evutil_socket_error_to_string(err));
+        }
+        if (events & (BEV_EVENT_EOF | BEV_EVENT_ERROR)) {
+          bufferevent_flush(bev, EV_WRITE, BEV_FINISHED);
+          bufferevent_free(bev);
+        }
+      };
+
+      struct bufferevent* bev = this->bev.release();
+      bufferevent_setcb(bev, nullptr, on_output, on_error, bev);
+      bufferevent_disable(bev, EV_READ);
+    }
+  }
+
+  memset(&this->local_addr, 0, sizeof(this->local_addr));
+  memset(&this->remote_addr, 0, sizeof(this->remote_addr));
+  this->virtual_network_id = false;
+  this->crypt_in.reset();
+  this->crypt_out.reset();
+}
+
+Channel::Message Channel::recv() {
+  struct evbuffer* buf = bufferevent_get_input(this->bev.get());
+
+  size_t header_size = (this->version == Version::BB_V4) ? 8 : 4;
+  PSOCommandHeader header;
+  if (evbuffer_copyout(buf, &header, header_size) < static_cast<ssize_t>(header_size)) {
+    throw out_of_range("no command available");
+  }
+
+  if (this->crypt_in.get()) {
+    this->crypt_in->decrypt(&header, header_size, false);
+  }
+
+  size_t command_logical_size = header.size(version);
+
+  // If encryption is enabled, BB pads commands to 8-byte boundaries, and this
+  // is not reflected in the size field. This logic does not occur if encryption
+  // is not yet enabled.
+  size_t command_physical_size = (this->crypt_in.get() && (version == Version::BB_V4))
+      ? ((command_logical_size + 7) & ~7)
+      : command_logical_size;
+  if (evbuffer_get_length(buf) < command_physical_size) {
+    throw out_of_range("no command available");
+  }
+
+  // If we get here, then there is a full command in the buffer. Some encryption
+  // algorithms' advancement depends on the decrypted data, so we have to
+  // actually decrypt the header again (with advance=true) to keep them in a
+  // consistent state.
+
+  string header_data(header_size, '\0');
+  if (evbuffer_remove(buf, header_data.data(), header_data.size()) < static_cast<ssize_t>(header_data.size())) {
+    throw logic_error("enough bytes available, but could not remove them");
+  }
+  if (this->crypt_in.get()) {
+    this->crypt_in->decrypt(header_data.data(), header_data.size());
+  }
+
+  string command_data(command_physical_size - header_size, '\0');
+  if (evbuffer_remove(buf, command_data.data(), command_data.size()) < static_cast<ssize_t>(command_data.size())) {
+    throw logic_error("enough bytes available, but could not remove them");
+  }
+
+  if (this->crypt_in.get()) {
+    // Some versions of PSO DC can send commands whose sizes are not a multiple
+    // of 4, but the server is expected to always use a multiple of 4 bytes when
+    // decrypting (the extra cipher bytes are lost). To emulate this behavior,
+    // we have to round up the size for DC commands here.
+    size_t orig_size = command_data.size();
+    command_data.resize((orig_size + 3) & (~3), 0);
+    this->crypt_in->decrypt(command_data.data(), command_data.size());
+    command_data.resize(orig_size);
+  }
+  command_data.resize(command_logical_size - header_size);
+
+  if (command_data_log.should_log(phosg::LogLevel::INFO) && (this->terminal_recv_color != phosg::TerminalFormat::END)) {
+    if (use_terminal_colors && this->terminal_recv_color != phosg::TerminalFormat::NORMAL) {
+      print_color_escape(stderr, this->terminal_recv_color, phosg::TerminalFormat::BOLD, phosg::TerminalFormat::END);
+    }
+
+    if (version == Version::BB_V4) {
+      command_data_log.info(
+          "Received from %s (version=BB command=%04hX flag=%08" PRIX32 ")",
+          this->name.c_str(),
+          header.command(this->version),
+          header.flag(this->version));
+    } else {
+      command_data_log.info(
+          "Received from %s (version=%s command=%02hX flag=%02" PRIX32 ")",
+          this->name.c_str(),
+          phosg::name_for_enum(this->version),
+          header.command(this->version),
+          header.flag(this->version));
+    }
+
+    vector<struct iovec> iovs;
+    iovs.emplace_back(iovec{.iov_base = header_data.data(), .iov_len = header_data.size()});
+    iovs.emplace_back(iovec{.iov_base = command_data.data(), .iov_len = command_data.size()});
+    phosg::print_data(stderr, iovs, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+
+    if (use_terminal_colors && this->terminal_recv_color != phosg::TerminalFormat::NORMAL) {
+      phosg::print_color_escape(stderr, phosg::TerminalFormat::NORMAL, phosg::TerminalFormat::END);
+    }
+  }
+
+  return {
+      .command = header.command(this->version),
+      .flag = header.flag(this->version),
+      .data = std::move(command_data),
+  };
+}
+
+void Channel::send(uint16_t cmd, uint32_t flag, bool silent) {
+  this->send(cmd, flag, nullptr, 0, silent);
+}
+
+void Channel::send(uint16_t cmd, uint32_t flag, const std::vector<std::pair<const void*, size_t>> blocks, bool silent) {
+  if (!this->connected()) {
+    channel_exceptions_log.warning("Attempted to send command on closed channel; dropping data");
+    return;
+  }
+
+  size_t size = 0;
+  for (const auto& b : blocks) {
+    size += b.second;
+  }
+
+  string send_data;
+  size_t logical_size;
+  size_t send_data_size = 0;
+  switch (this->version) {
+    case Version::DC_NTE:
+    case Version::DC_V1_11_2000_PROTOTYPE:
+    case Version::DC_V1:
+    case Version::DC_V2:
+    case Version::GC_NTE:
+    case Version::GC_V3:
+    case Version::GC_EP3_NTE:
+    case Version::GC_EP3:
+    case Version::XB_V3: {
+      PSOCommandHeaderDCV3 header;
+      if (this->crypt_out.get() &&
+          (this->version != Version::DC_NTE) &&
+          (this->version != Version::DC_V1_11_2000_PROTOTYPE) &&
+          (this->version != Version::DC_V1)) {
+        send_data_size = (sizeof(header) + size + 3) & ~3;
+      } else {
+        send_data_size = (sizeof(header) + size);
+      }
+      logical_size = send_data_size;
+      header.command = cmd;
+      header.flag = flag;
+      header.size = send_data_size;
+      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
+      break;
+    }
+    case Version::PC_PATCH:
+    case Version::BB_PATCH:
+    case Version::PC_NTE:
+    case Version::PC_V2: {
+      PSOCommandHeaderPC header;
+      if (this->crypt_out.get()) {
+        send_data_size = (sizeof(header) + size + 3) & ~3;
+      } else {
+        send_data_size = (sizeof(header) + size);
+      }
+      logical_size = send_data_size;
+      header.size = send_data_size;
+      header.command = cmd;
+      header.flag = flag;
+      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
+      break;
+    }
+    case Version::BB_V4: {
+      // BB has an annoying behavior here: command lengths must be multiples of
+      // 4, but the actual data length must be a multiple of 8. If the size
+      // field is not divisible by 8, 4 extra bytes are sent anyway. This
+      // behavior only applies when encryption is enabled - any commands sent
+      // before encryption is enabled have no size restrictions (except they
+      // must include a full header and must fit in the client's receive
+      // buffer), and no implicit extra bytes are sent.
+      PSOCommandHeaderBB header;
+      if (this->crypt_out.get()) {
+        send_data_size = (sizeof(header) + size + 7) & ~7;
+      } else {
+        send_data_size = (sizeof(header) + size);
+      }
+      logical_size = (sizeof(header) + size + 3) & ~3;
+      header.size = logical_size;
+      header.command = cmd;
+      header.flag = flag;
+      send_data.append(reinterpret_cast<const char*>(&header), sizeof(header));
+      break;
+    }
+
+    default:
+      throw logic_error("unimplemented game version in send_command");
+  }
+
+  // All versions of PSO I've seen (so far) have a receive buffer 0x7C00
+  // bytes in size
+  if (send_data_size > 0x7C00) {
+    throw runtime_error("outbound command too large");
+  }
+
+  send_data.reserve(send_data_size);
+  for (const auto& b : blocks) {
+    send_data.append(reinterpret_cast<const char*>(b.first), b.second);
+  }
+  send_data.resize(send_data_size, '\0');
+
+  if (!silent && (command_data_log.should_log(phosg::LogLevel::INFO)) && (this->terminal_send_color != phosg::TerminalFormat::END)) {
+    if (use_terminal_colors && this->terminal_send_color != phosg::TerminalFormat::NORMAL) {
+      print_color_escape(stderr, phosg::TerminalFormat::FG_YELLOW, phosg::TerminalFormat::BOLD, phosg::TerminalFormat::END);
+    }
+    if (version == Version::BB_V4) {
+      command_data_log.info("Sending to %s (version=BB command=%04hX flag=%08" PRIX32 ")",
+          this->name.c_str(), cmd, flag);
+    } else {
+      command_data_log.info("Sending to %s (version=%s command=%02hX flag=%02" PRIX32 ")",
+          this->name.c_str(), phosg::name_for_enum(version), cmd, flag);
+    }
+    phosg::print_data(stderr, send_data.data(), logical_size, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+    if (use_terminal_colors && this->terminal_send_color != phosg::TerminalFormat::NORMAL) {
+      print_color_escape(stderr, phosg::TerminalFormat::NORMAL, phosg::TerminalFormat::END);
+    }
+  }
+
+  if (this->crypt_out.get()) {
+    this->crypt_out->encrypt(send_data.data(), send_data.size());
+  }
+
+  struct evbuffer* buf = bufferevent_get_output(this->bev.get());
+  evbuffer_add(buf, send_data.data(), send_data.size());
+}
+
+void Channel::send(uint16_t cmd, uint32_t flag, const void* data, size_t size, bool silent) {
+  this->send(cmd, flag, {make_pair(data, size)}, silent);
+}
+
+void Channel::send(uint16_t cmd, uint32_t flag, const string& data, bool silent) {
+  this->send(cmd, flag, data.data(), data.size(), silent);
+}
+
+void Channel::send(const void* data, size_t size, bool silent) {
+  size_t header_size = (this->version == Version::BB_V4) ? 8 : 4;
+  const auto* header = reinterpret_cast<const PSOCommandHeader*>(data);
+  this->send(
+      header->command(this->version),
+      header->flag(this->version),
+      reinterpret_cast<const uint8_t*>(data) + header_size,
+      size - header_size,
+      silent);
+}
+
+void Channel::send(const string& data, bool silent) {
+  return this->send(data.data(), data.size(), silent);
+}
+
+void Channel::dispatch_on_input(struct bufferevent*, void* ctx) {
+  Channel* ch = reinterpret_cast<Channel*>(ctx);
+  // The client can be disconnected during on_command_received, so we have to
+  // make sure ch->bev is valid every time before calling recv()
+  while (ch->bev.get()) {
+    Message msg;
+    try {
+      msg = ch->recv();
+    } catch (const out_of_range&) {
+      break;
+    } catch (const exception& e) {
+      channel_exceptions_log.warning("Error receiving on channel: %s", e.what());
+      ch->on_error(*ch, BEV_EVENT_ERROR);
+      break;
+    }
+    if (ch->on_command_received) {
+      ch->on_command_received(*ch, msg.command, msg.flag, msg.data);
+    }
+  }
+}
+
+void Channel::dispatch_on_error(struct bufferevent*, short events, void* ctx) {
+  Channel* ch = reinterpret_cast<Channel*>(ctx);
+  if (ch->on_error) {
+    ch->on_error(*ch, events);
+  } else {
+    ch->disconnect();
+  }
+}

src/Channel.hh

@@ -1,102 +1,103 @@
-#pragma once
-
-#include <netinet/in.h>
-
-#include <memory>
-#include <string>
-
-#include "PSOEncryption.hh"
-#include "PSOProtocol.hh"
-#include "Version.hh"
-
-struct Channel {
-  std::unique_ptr<struct bufferevent, void (*)(struct bufferevent*)> bev;
-  struct sockaddr_storage local_addr;
-  struct sockaddr_storage remote_addr;
-  bool is_virtual_connection;
-
-  Version version;
-  uint8_t language;
-  std::shared_ptr<PSOEncryption> crypt_in;
-  std::shared_ptr<PSOEncryption> crypt_out;
-
-  std::string name;
-  TerminalFormat terminal_send_color;
-  TerminalFormat terminal_recv_color;
-
-  struct Message {
-    uint16_t command;
-    uint32_t flag;
-    std::string data;
-  };
-
-  typedef void (*on_command_received_t)(Channel&, uint16_t, uint32_t, std::string&);
-  typedef void (*on_error_t)(Channel&, short);
-
-  on_command_received_t on_command_received;
-  on_error_t on_error;
-  void* context_obj;
-
-  // Creates an unconnected channel
-  Channel(
-      Version version,
-      uint8_t language,
-      on_command_received_t on_command_received,
-      on_error_t on_error,
-      void* context_obj,
-      const std::string& name,
-      TerminalFormat terminal_send_color = TerminalFormat::END,
-      TerminalFormat terminal_recv_color = TerminalFormat::END);
-  // Creates a connected channel
-  Channel(
-      struct bufferevent* bev,
-      Version version,
-      uint8_t language,
-      on_command_received_t on_command_received,
-      on_error_t on_error,
-      void* context_obj,
-      const std::string& name = "",
-      TerminalFormat terminal_send_color = TerminalFormat::END,
-      TerminalFormat terminal_recv_color = TerminalFormat::END);
-  Channel(const Channel& other) = delete;
-  Channel(Channel&& other) = delete;
-  Channel& operator=(const Channel& other) = delete;
-  Channel& operator=(Channel&& other) = delete;
-
-  void replace_with(
-      Channel&& other,
-      on_command_received_t on_command_received,
-      on_error_t on_error,
-      void* context_obj,
-      const std::string& name = "");
-
-  void set_bufferevent(struct bufferevent* bev);
-
-  inline bool connected() const {
-    return this->bev.get() != nullptr;
-  }
-  void disconnect();
-
-  // Receives a message. Throws std::out_of_range if no messages are available.
-  Message recv();
-
-  // Sends a message with an automatically-constructed header.
-  void send(uint16_t cmd, uint32_t flag = 0, bool silent = false);
-  void send(uint16_t cmd, uint32_t flag, const void* data, size_t size, bool silent = false);
-  void send(uint16_t cmd, uint32_t flag, const std::vector<std::pair<const void*, size_t>> blocks, bool silent = false);
-  void send(uint16_t cmd, uint32_t flag, const std::string& data, bool silent = false);
-  template <typename CmdT>
-    requires(!std::is_pointer_v<CmdT>)
-  void send(uint16_t cmd, uint32_t flag, const CmdT& data, bool silent = false) {
-    this->send(cmd, flag, &data, sizeof(data), silent);
-  }
-
-  // Sends a message with a pre-existing header (as the first few bytes in the
-  // data)
-  void send(const void* data, size_t size, bool silent = false);
-  void send(const std::string& data, bool silent = false);
-
-private:
-  static void dispatch_on_input(struct bufferevent*, void* ctx);
-  static void dispatch_on_error(struct bufferevent*, short events, void* ctx);
-};
+#pragma once
+
+#include <netinet/in.h>
+
+#include <memory>
+#include <string>
+
+#include "PSOEncryption.hh"
+#include "PSOProtocol.hh"
+#include "Version.hh"
+
+struct Channel {
+  std::unique_ptr<struct bufferevent, void (*)(struct bufferevent*)> bev;
+  struct sockaddr_storage local_addr;
+  struct sockaddr_storage remote_addr;
+  uint64_t virtual_network_id; // 0 = normal TCP connection
+
+  Version version;
+  uint8_t language;
+  std::shared_ptr<PSOEncryption> crypt_in;
+  std::shared_ptr<PSOEncryption> crypt_out;
+
+  std::string name;
+  phosg::TerminalFormat terminal_send_color;
+  phosg::TerminalFormat terminal_recv_color;
+
+  struct Message {
+    uint16_t command;
+    uint32_t flag;
+    std::string data;
+  };
+
+  typedef void (*on_command_received_t)(Channel&, uint16_t, uint32_t, std::string&);
+  typedef void (*on_error_t)(Channel&, short);
+
+  on_command_received_t on_command_received;
+  on_error_t on_error;
+  void* context_obj;
+
+  // Creates an unconnected channel
+  Channel(
+      Version version,
+      uint8_t language,
+      on_command_received_t on_command_received,
+      on_error_t on_error,
+      void* context_obj,
+      const std::string& name,
+      phosg::TerminalFormat terminal_send_color = phosg::TerminalFormat::END,
+      phosg::TerminalFormat terminal_recv_color = phosg::TerminalFormat::END);
+  // Creates a connected channel
+  Channel(
+      struct bufferevent* bev,
+      uint64_t virtual_network_id,
+      Version version,
+      uint8_t language,
+      on_command_received_t on_command_received,
+      on_error_t on_error,
+      void* context_obj,
+      const std::string& name = "",
+      phosg::TerminalFormat terminal_send_color = phosg::TerminalFormat::END,
+      phosg::TerminalFormat terminal_recv_color = phosg::TerminalFormat::END);
+  Channel(const Channel& other) = delete;
+  Channel(Channel&& other) = delete;
+  Channel& operator=(const Channel& other) = delete;
+  Channel& operator=(Channel&& other) = delete;
+
+  void replace_with(
+      Channel&& other,
+      on_command_received_t on_command_received,
+      on_error_t on_error,
+      void* context_obj,
+      const std::string& name = "");
+
+  void set_bufferevent(struct bufferevent* bev, uint64_t virtual_network_id);
+
+  inline bool connected() const {
+    return this->bev.get() != nullptr;
+  }
+  void disconnect();
+
+  // Receives a message. Throws std::out_of_range if no messages are available.
+  Message recv();
+
+  // Sends a message with an automatically-constructed header.
+  void send(uint16_t cmd, uint32_t flag = 0, bool silent = false);
+  void send(uint16_t cmd, uint32_t flag, const void* data, size_t size, bool silent = false);
+  void send(uint16_t cmd, uint32_t flag, const std::vector<std::pair<const void*, size_t>> blocks, bool silent = false);
+  void send(uint16_t cmd, uint32_t flag, const std::string& data, bool silent = false);
+  template <typename CmdT>
+    requires(!std::is_pointer_v<CmdT>)
+  void send(uint16_t cmd, uint32_t flag, const CmdT& data, bool silent = false) {
+    this->send(cmd, flag, &data, sizeof(data), silent);
+  }
+
+  // Sends a message with a pre-existing header (as the first few bytes in the
+  // data)
+  void send(const void* data, size_t size, bool silent = false);
+  void send(const std::string& data, bool silent = false);
+
+private:
+  static void dispatch_on_input(struct bufferevent*, void* ctx);
+  static void dispatch_on_error(struct bufferevent*, short events, void* ctx);
+};

src/ChatCommands.hh

@@ -1,14 +1,14 @@
-#pragma once
-
-#include <stdint.h>
-
-#include <memory>
-#include <string>
-
-#include "Client.hh"
-#include "Lobby.hh"
-#include "ProxyServer.hh"
-#include "ServerState.hh"
-
-void on_chat_command(std::shared_ptr<Client> c, const std::string& text);
-void on_chat_command(std::shared_ptr<ProxyServer::LinkedSession> ses, const std::string& text);
+#pragma once
+
+#include <stdint.h>
+
+#include <memory>
+#include <string>
+
+#include "Client.hh"
+#include "Lobby.hh"
+#include "ProxyServer.hh"
+#include "ServerState.hh"
+
+void on_chat_command(std::shared_ptr<Client> c, const std::string& text);
+void on_chat_command(std::shared_ptr<ProxyServer::LinkedSession> ses, const std::string& text);

src/ChoiceSearch.cc

@@ -1,150 +1,150 @@
-#include "ChoiceSearch.hh"
-
-#include <inttypes.h>
-#include <string.h>
-
-#include "Client.hh"
-
-using namespace std;
-
-const vector<ChoiceSearchCategory> CHOICE_SEARCH_CATEGORIES({
-    ChoiceSearchCategory{
-        .id = 0x0001,
-        .name = "Level",
-        .choices = {
-            {0x0000, "Any"},
-            {0x0001, "Own level +/- 5"},
-            {0x0002, "Level 1-10"},
-            {0x0003, "Level 11-20"},
-            {0x0004, "Level 21-40"},
-            {0x0005, "Level 41-60"},
-            {0x0006, "Level 61-80"},
-            {0x0007, "Level 81-100"},
-            {0x0008, "Level 101-120"},
-            {0x0009, "Level 121-160"},
-            {0x000A, "Level 161-200"},
-        },
-        .client_matches = +[](shared_ptr<Client> searcher_c, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
-          if (choice_id == 0x0000) {
-            return true;
-          }
-          uint32_t target_level = target_c->character()->disp.stats.level + 1;
-          switch (choice_id) {
-            case 0x0001:
-              return (labs(static_cast<int32_t>(target_level - searcher_c->character()->disp.stats.level)) <= 5);
-            case 0x0002:
-              return (target_level <= 10);
-            case 0x0003:
-              return (target_level > 10) && (target_level <= 20);
-            case 0x0004:
-              return (target_level > 20) && (target_level <= 40);
-            case 0x0005:
-              return (target_level > 40) && (target_level <= 60);
-            case 0x0006:
-              return (target_level > 60) && (target_level <= 80);
-            case 0x0007:
-              return (target_level > 80) && (target_level <= 100);
-            case 0x0008:
-              return (target_level > 100) && (target_level <= 120);
-            case 0x0009:
-              return (target_level > 120) && (target_level <= 160);
-            case 0x000A:
-              return (target_level > 160) && (target_level <= 200);
-          }
-          return false;
-        },
-    },
-    ChoiceSearchCategory{
-        .id = 0x0002,
-        .name = "Class",
-        .choices = {
-            {0x0000, "Any"},
-            {0x0010, "Hunter"},
-            {0x0001, "HUmar"},
-            {0x0002, "HUnewearl"},
-            {0x0003, "HUcast"},
-            {0x000A, "HUcaseal"},
-            {0x0011, "Ranger"},
-            {0x0004, "RAmar"},
-            {0x000C, "RAmarl"},
-            {0x0005, "RAcast"},
-            {0x0006, "RAcaseal"},
-            {0x0012, "Force"},
-            {0x000B, "FOmar"},
-            {0x0007, "FOmarl"},
-            {0x0008, "FOnewm"},
-            {0x0009, "FOnewearl"},
-        },
-        .client_matches = +[](shared_ptr<Client>, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
-          switch (choice_id) {
-            case 0x0000:
-              return true;
-            case 0x0010:
-              return target_c->character()->disp.visual.class_flags & 0x20;
-            case 0x0011:
-              return target_c->character()->disp.visual.class_flags & 0x40;
-            case 0x0012:
-              return target_c->character()->disp.visual.class_flags & 0x80;
-            default:
-              return ((choice_id - 1) == target_c->character()->disp.visual.char_class);
-          }
-        },
-    },
-    ChoiceSearchCategory{
-        .id = 0x0003,
-        .name = "Platform",
-        .choices = {
-            {0x0000, "Any"},
-            {0x0001, "DC betas"},
-            {0x0002, "DC V1"},
-            {0x0003, "DC V2 / PC"},
-            {0x0004, "GC / Xbox Episodes 1&2"},
-            {0x0005, "GC Episode 3"},
-            {0x0006, "BB"},
-        },
-        .client_matches = +[](shared_ptr<Client>, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
-          if (choice_id == 0x0000) {
-            return true;
-          }
-          switch (target_c->version()) {
-            case Version::DC_NTE:
-            case Version::DC_V1_11_2000_PROTOTYPE:
-              return (choice_id == 0x0001);
-            case Version::DC_V1:
-              return (choice_id == 0x0002);
-            case Version::DC_V2:
-            case Version::PC_NTE:
-            case Version::PC_V2:
-              return (choice_id == 0x0003);
-            case Version::GC_NTE:
-            case Version::GC_V3:
-            case Version::XB_V3:
-              return (choice_id == 0x0004);
-            case Version::GC_EP3_NTE:
-            case Version::GC_EP3:
-              return (choice_id == 0x0005);
-            case Version::BB_V4:
-              return (choice_id == 0x0006);
-            default:
-              return false;
-          }
-        },
-    },
-    ChoiceSearchCategory{
-        .id = 0x0204,
-        .name = "Game mode",
-        .choices = {
-            {0x0000, "Any"},
-            {0x0001, "Normal"},
-            {0x0002, "Hard"},
-            {0x0003, "Very Hard"},
-            {0x0004, "Ultimate"},
-            {0x0005, "Battle"},
-            {0x0006, "Challenge"},
-        },
-        .client_matches = +[](shared_ptr<Client>, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
-          uint16_t target_choice_id = target_c->character()->choice_search_config.get_setting(0x0204);
-          return (choice_id == 0) || (target_choice_id == 0) || (choice_id == target_choice_id);
-        },
-    },
-});
+#include "ChoiceSearch.hh"
+
+#include <inttypes.h>
+#include <string.h>
+
+#include "Client.hh"
+
+using namespace std;
+
+const vector<ChoiceSearchCategory> CHOICE_SEARCH_CATEGORIES({
+    ChoiceSearchCategory{
+        .id = 0x0001,
+        .name = "Level",
+        .choices = {
+            {0x0000, "Any"},
+            {0x0001, "Own level +/- 5"},
+            {0x0002, "Level 1-10"},
+            {0x0003, "Level 11-20"},
+            {0x0004, "Level 21-40"},
+            {0x0005, "Level 41-60"},
+            {0x0006, "Level 61-80"},
+            {0x0007, "Level 81-100"},
+            {0x0008, "Level 101-120"},
+            {0x0009, "Level 121-160"},
+            {0x000A, "Level 161-200"},
+        },
+        .client_matches = +[](shared_ptr<Client> searcher_c, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
+          if (choice_id == 0x0000) {
+            return true;
+          }
+          uint32_t target_level = target_c->character()->disp.stats.level + 1;
+          switch (choice_id) {
+            case 0x0001:
+              return (labs(static_cast<int32_t>(target_level - searcher_c->character()->disp.stats.level)) <= 5);
+            case 0x0002:
+              return (target_level <= 10);
+            case 0x0003:
+              return (target_level > 10) && (target_level <= 20);
+            case 0x0004:
+              return (target_level > 20) && (target_level <= 40);
+            case 0x0005:
+              return (target_level > 40) && (target_level <= 60);
+            case 0x0006:
+              return (target_level > 60) && (target_level <= 80);
+            case 0x0007:
+              return (target_level > 80) && (target_level <= 100);
+            case 0x0008:
+              return (target_level > 100) && (target_level <= 120);
+            case 0x0009:
+              return (target_level > 120) && (target_level <= 160);
+            case 0x000A:
+              return (target_level > 160) && (target_level <= 200);
+          }
+          return false;
+        },
+    },
+    ChoiceSearchCategory{
+        .id = 0x0002,
+        .name = "Class",
+        .choices = {
+            {0x0000, "Any"},
+            {0x0010, "Hunter"},
+            {0x0001, "HUmar"},
+            {0x0002, "HUnewearl"},
+            {0x0003, "HUcast"},
+            {0x000A, "HUcaseal"},
+            {0x0011, "Ranger"},
+            {0x0004, "RAmar"},
+            {0x000C, "RAmarl"},
+            {0x0005, "RAcast"},
+            {0x0006, "RAcaseal"},
+            {0x0012, "Force"},
+            {0x000B, "FOmar"},
+            {0x0007, "FOmarl"},
+            {0x0008, "FOnewm"},
+            {0x0009, "FOnewearl"},
+        },
+        .client_matches = +[](shared_ptr<Client>, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
+          switch (choice_id) {
+            case 0x0000:
+              return true;
+            case 0x0010:
+              return target_c->character()->disp.visual.class_flags & 0x20;
+            case 0x0011:
+              return target_c->character()->disp.visual.class_flags & 0x40;
+            case 0x0012:
+              return target_c->character()->disp.visual.class_flags & 0x80;
+            default:
+              return ((choice_id - 1) == target_c->character()->disp.visual.char_class);
+          }
+        },
+    },
+    ChoiceSearchCategory{
+        .id = 0x0003,
+        .name = "Platform",
+        .choices = {
+            {0x0000, "Any"},
+            {0x0001, "DC betas"},
+            {0x0002, "DC V1"},
+            {0x0003, "DC V2 / PC"},
+            {0x0004, "GC / Xbox Episodes 1&2"},
+            {0x0005, "GC Episode 3"},
+            {0x0006, "BB"},
+        },
+        .client_matches = +[](shared_ptr<Client>, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
+          if (choice_id == 0x0000) {
+            return true;
+          }
+          switch (target_c->version()) {
+            case Version::DC_NTE:
+            case Version::DC_V1_11_2000_PROTOTYPE:
+              return (choice_id == 0x0001);
+            case Version::DC_V1:
+              return (choice_id == 0x0002);
+            case Version::DC_V2:
+            case Version::PC_NTE:
+            case Version::PC_V2:
+              return (choice_id == 0x0003);
+            case Version::GC_NTE:
+            case Version::GC_V3:
+            case Version::XB_V3:
+              return (choice_id == 0x0004);
+            case Version::GC_EP3_NTE:
+            case Version::GC_EP3:
+              return (choice_id == 0x0005);
+            case Version::BB_V4:
+              return (choice_id == 0x0006);
+            default:
+              return false;
+          }
+        },
+    },
+    ChoiceSearchCategory{
+        .id = 0x0204,
+        .name = "Game mode",
+        .choices = {
+            {0x0000, "Any"},
+            {0x0001, "Normal"},
+            {0x0002, "Hard"},
+            {0x0003, "Very Hard"},
+            {0x0004, "Ultimate"},
+            {0x0005, "Battle"},
+            {0x0006, "Challenge"},
+        },
+        .client_matches = +[](shared_ptr<Client>, shared_ptr<Client> target_c, uint16_t choice_id) -> bool {
+          uint16_t target_choice_id = target_c->character()->choice_search_config.get_setting(0x0204);
+          return (choice_id == 0) || (target_choice_id == 0) || (choice_id == target_choice_id);
+        },
+    },
+});

src/ChoiceSearch.hh

@@ -1,43 +1,62 @@
-#pragma once
-
-#include <functional>
-#include <memory>
-#include <phosg/Encoding.hh>
-#include <string>
-#include <vector>
-
-#include "Text.hh"
-
-class Client;
-
-struct ChoiceSearchConfig {
-  le_uint32_t disabled = 1; // 0 = enabled, 1 = disabled. Unused in command C3
-  struct Entry {
-    le_uint16_t parent_choice_id = 0;
-    le_uint16_t choice_id = 0;
-  } __attribute__((packed));
-  parray<Entry, 5> entries;
-
-  int32_t get_setting(uint16_t parent_choice_id) const {
-    for (size_t z = 0; z < this->entries.size(); z++) {
-      if (this->entries[z].parent_choice_id == parent_choice_id) {
-        return this->entries[z].choice_id;
-      }
-    }
-    return -1;
-  }
-} __attribute__((packed));
-
-struct ChoiceSearchCategory {
-  struct Choice {
-    uint16_t id;
-    const char* name;
-  };
-
-  uint16_t id;
-  const char* name;
-  std::vector<Choice> choices;
-  std::function<bool(std::shared_ptr<Client> searcher_c, std::shared_ptr<Client> target_c, uint16_t choice_id)> client_matches;
-};
-
-extern const std::vector<ChoiceSearchCategory> CHOICE_SEARCH_CATEGORIES;
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <phosg/Encoding.hh>
+#include <string>
+#include <vector>
+
+#include "Text.hh"
+#include "Types.hh"
+
+class Client;
+
+template <bool BE>
+struct ChoiceSearchConfigT {
+  U32T<BE> disabled = 1; // 0 = enabled, 1 = disabled. Unused in command C3
+  struct Entry {
+    U16T<BE> parent_choice_id = 0;
+    U16T<BE> choice_id = 0;
+  } __packed_ws__(Entry, 4);
+  parray<Entry, 5> entries;
+
+  int32_t get_setting(uint16_t parent_choice_id) const {
+    for (size_t z = 0; z < this->entries.size(); z++) {
+      if (this->entries[z].parent_choice_id == parent_choice_id) {
+        return this->entries[z].choice_id;
+      }
+    }
+    return -1;
+  }
+
+  operator ChoiceSearchConfigT<!BE>() const {
+    ChoiceSearchConfigT<!BE> ret;
+    ret.disabled = this->disabled.load();
+    for (size_t z = 0; z < this->entries.size(); z++) {
+      auto& ret_e = ret.entries[z];
+      const auto& this_e = this->entries[z];
+      ret_e.parent_choice_id = this_e.parent_choice_id.load();
+      ret_e.choice_id = this_e.choice_id.load();
+    }
+    return ret;
+  }
+} __packed__;
+
+using ChoiceSearchConfig = ChoiceSearchConfigT<false>;
+using ChoiceSearchConfigBE = ChoiceSearchConfigT<true>;
+check_struct_size(ChoiceSearchConfig, 0x18);
+check_struct_size(ChoiceSearchConfigBE, 0x18);
+
+struct ChoiceSearchCategory {
+  struct Choice {
+    uint16_t id;
+    const char* name;
+  };
+
+  uint16_t id;
+  const char* name;
+  std::vector<Choice> choices;
+  std::function<bool(std::shared_ptr<Client> searcher_c, std::shared_ptr<Client> target_c, uint16_t choice_id)> client_matches;
+};
+
+extern const std::vector<ChoiceSearchCategory> CHOICE_SEARCH_CATEGORIES;

src/Client.hh

@@ -1,373 +1,420 @@
-#pragma once
-
-#include <netinet/in.h>
-
-#include <memory>
-#include <stdexcept>
-
-#include "Channel.hh"
-#include "CommandFormats.hh"
-#include "Episode3/BattleRecord.hh"
-#include "Episode3/Tournament.hh"
-#include "FileContentsCache.hh"
-#include "FunctionCompiler.hh"
-#include "License.hh"
-#include "PSOEncryption.hh"
-#include "PSOProtocol.hh"
-#include "PatchFileIndex.hh"
-#include "Quest.hh"
-#include "QuestScript.hh"
-#include "TeamIndex.hh"
-#include "Text.hh"
-
-extern const uint64_t CLIENT_CONFIG_MAGIC;
-
-class Server;
-struct Lobby;
-
-class Client : public std::enable_shared_from_this<Client> {
-public:
-  enum class Flag : uint64_t {
-    // clang-format off
-
-    // This mask specifies which flags are sent to the client
-    // TODO: It'd be nice to use a pattern here (e.g. all server-side flags are
-    // in the high bits) but that would require re-recording or manually
-    // rewriting all the tests
-    CLIENT_SIDE_MASK                    = 0xFFFFFFFFFC0FFFFB,
-
-    // Version-related flags
-    CHECKED_FOR_DC_V1_PROTOTYPE         = 0x0000000000000002,
-    LICENSE_WAS_CREATED                 = 0x0000000000000004, // Server-side only
-    NO_D6_AFTER_LOBBY                   = 0x0000000000000100,
-    NO_D6                               = 0x0000000000000200,
-    FORCE_ENGLISH_LANGUAGE_BB           = 0x0000000000000400,
-
-    // Flags describing the behavior for send_function_call
-    NO_SEND_FUNCTION_CALL               = 0x0000000000001000,
-    ENCRYPTED_SEND_FUNCTION_CALL        = 0x0000000000002000,
-    SEND_FUNCTION_CALL_CHECKSUM_ONLY    = 0x0000000000004000,
-    SEND_FUNCTION_CALL_NO_CACHE_PATCH   = 0x0000000000008000,
-    USE_OVERFLOW_FOR_SEND_FUNCTION_CALL = 0x0000000000010000,
-
-    // State flags
-    LOADING                             = 0x0000000000100000, // Server-side only
-    LOADING_QUEST                       = 0x0000000000200000, // Server-side only
-    LOADING_RUNNING_JOINABLE_QUEST      = 0x0000000000400000, // Server-side only
-    LOADING_TOURNAMENT                  = 0x0000000000800000, // Server-side only
-    IN_INFORMATION_MENU                 = 0x0000000001000000, // Server-side only
-    AT_WELCOME_MESSAGE                  = 0x0000000002000000, // Server-side only
-    SAVE_ENABLED                        = 0x0000000004000000,
-    HAS_EP3_CARD_DEFS                   = 0x0000000008000000,
-    HAS_EP3_MEDIA_UPDATES               = 0x0000000010000000,
-    USE_OVERRIDE_RANDOM_SEED            = 0x0000000020000000,
-    HAS_GUILD_CARD_NUMBER               = 0x0000000040000000,
-    AT_BANK_COUNTER                     = 0x0000000080000000, // Server-side only
-    SHOULD_SEND_ARTIFICIAL_ITEM_STATE   = 0x0001000000000000, // Server-side only
-    SHOULD_SEND_ARTIFICIAL_FLAG_STATE   = 0x0002000000000000, // Server-side only
-    SHOULD_SEND_ENABLE_SAVE             = 0x0004000000000000,
-    SWITCH_ASSIST_ENABLED               = 0x0000000100000000,
-
-    // Cheat mode and option flags
-    INFINITE_HP_ENABLED                 = 0x0000000200000000,
-    INFINITE_TP_ENABLED                 = 0x0000000400000000,
-    DEBUG_ENABLED                       = 0x0000000800000000,
-    RARE_DROP_NOTIFICATIONS_ENABLED     = 0x0010000000000000,
-    ALL_DROP_NOTIFICATIONS_ENABLED      = 0x0020000000000000,
-
-    // Proxy option flags
-    PROXY_SAVE_FILES                    = 0x0000001000000000,
-    PROXY_CHAT_COMMANDS_ENABLED         = 0x0000002000000000,
-    PROXY_PLAYER_NOTIFICATIONS_ENABLED  = 0x0000008000000000,
-    PROXY_SUPPRESS_CLIENT_PINGS         = 0x0000010000000000,
-    PROXY_SUPPRESS_REMOTE_LOGIN         = 0x0000020000000000,
-    PROXY_ZERO_REMOTE_GUILD_CARD        = 0x0000040000000000,
-    PROXY_EP3_INFINITE_MESETA_ENABLED   = 0x0000080000000000,
-    PROXY_EP3_INFINITE_TIME_ENABLED     = 0x0000100000000000,
-    PROXY_RED_NAME_ENABLED              = 0x0000200000000000,
-    PROXY_BLANK_NAME_ENABLED            = 0x0000400000000000,
-    PROXY_BLOCK_FUNCTION_CALLS          = 0x0000800000000000,
-    PROXY_EP3_UNMASK_WHISPERS           = 0x0008000000000000,
-    // clang-format on
-  };
-
-  static constexpr uint64_t DEFAULT_FLAGS = static_cast<uint64_t>(Flag::PROXY_CHAT_COMMANDS_ENABLED);
-
-  struct Config {
-    uint64_t enabled_flags = DEFAULT_FLAGS; // Client::Flag enum
-    uint32_t specific_version = 0;
-    int32_t override_random_seed = 0;
-    uint8_t override_section_id = 0xFF; // FF = no override
-    uint8_t override_lobby_event = 0xFF; // FF = no override
-    uint8_t override_lobby_number = 0x80; // 80 = no override
-    uint32_t proxy_destination_address = 0;
-    uint16_t proxy_destination_port = 0;
-
-    Config() = default;
-
-    bool operator==(const Config& other) const = default;
-    bool operator!=(const Config& other) const = default;
-
-    bool should_update_vs(const Config& other) const;
-
-    [[nodiscard]] static inline bool check_flag(uint64_t enabled_flags, Flag flag) {
-      return !!(enabled_flags & static_cast<uint64_t>(flag));
-    }
-
-    [[nodiscard]] inline bool check_flag(Flag flag) const {
-      return this->check_flag(this->enabled_flags, flag);
-    }
-    inline void set_flag(Flag flag) {
-      this->enabled_flags |= static_cast<uint64_t>(flag);
-    }
-    inline void clear_flag(Flag flag) {
-      this->enabled_flags &= (~static_cast<uint64_t>(flag));
-    }
-    inline void toggle_flag(Flag flag) {
-      this->enabled_flags ^= static_cast<uint64_t>(flag);
-    }
-
-    void set_flags_for_version(Version version, int64_t sub_version);
-
-    template <size_t Bytes>
-    void parse_from(const parray<uint8_t, Bytes>& data) {
-      StringReader r(data.data(), data.size());
-      if (r.get_u32l() != CLIENT_CONFIG_MAGIC) {
-        throw std::invalid_argument("config signature is incorrect");
-      }
-      this->specific_version = r.get_u32l();
-      this->enabled_flags = r.get_u64l();
-      this->override_random_seed = r.get_u32l();
-      this->proxy_destination_address = r.get_u32b();
-      this->proxy_destination_port = r.get_u16l();
-      this->override_section_id = r.get_u8();
-      this->override_lobby_event = r.get_u8();
-      this->override_lobby_number = r.get_u8();
-    }
-
-    template <size_t Bytes>
-    void serialize_into(parray<uint8_t, Bytes>& data) const {
-      StringWriter w;
-      w.put_u32l(CLIENT_CONFIG_MAGIC);
-      w.put_u32l(this->specific_version);
-      w.put_u64l(this->enabled_flags & static_cast<uint64_t>(Flag::CLIENT_SIDE_MASK));
-      w.put_u32l(this->override_random_seed);
-      w.put_u32b(this->proxy_destination_address);
-      w.put_u16l(this->proxy_destination_port);
-      w.put_u8(this->override_section_id);
-      w.put_u8(this->override_lobby_event);
-      w.put_u8(this->override_lobby_number);
-
-      const auto& s = w.str();
-      for (size_t z = 0; z < s.size(); z++) {
-        data[z] = s[z];
-      }
-      data.clear_after(s.size(), 0xFF);
-    }
-  };
-
-  std::weak_ptr<Server> server;
-  uint64_t id;
-  PrefixedLogger log;
-
-  // License & account
-  std::shared_ptr<License> license;
-
-  // Network
-  Channel channel;
-  struct sockaddr_storage next_connection_addr;
-  ServerBehavior server_behavior;
-  bool should_disconnect;
-  bool should_send_to_lobby_server;
-  bool should_send_to_proxy_server;
-  std::unordered_map<std::string, std::function<void()>> disconnect_hooks;
-  std::shared_ptr<XBNetworkLocation> xb_netloc;
-  parray<le_uint32_t, 3> xb_9E_unknown_a1a;
-  uint8_t bb_connection_phase;
-  uint64_t ping_start_time;
-
-  // Patch server
-  std::vector<PatchFileChecksumRequest> patch_file_checksum_requests;
-
-  // Lobby/positioning
-  Config config;
-  Config synced_config;
-  int32_t sub_version;
-  float x;
-  float z;
-  uint32_t floor;
-  std::weak_ptr<Lobby> lobby;
-  uint8_t lobby_client_id;
-  uint8_t lobby_arrow_color;
-  int64_t preferred_lobby_id; // <0 = no preference
-
-  std::unique_ptr<struct event, void (*)(struct event*)> save_game_data_event;
-  std::unique_ptr<struct event, void (*)(struct event*)> send_ping_event;
-  std::unique_ptr<struct event, void (*)(struct event*)> idle_timeout_event;
-  int16_t card_battle_table_number;
-  uint16_t card_battle_table_seat_number;
-  uint16_t card_battle_table_seat_state;
-  std::weak_ptr<Episode3::Tournament::Team> ep3_tournament_team;
-  std::shared_ptr<Episode3::BattleRecord> ep3_prev_battle_record;
-  std::shared_ptr<const Menu> last_menu_sent;
-  struct JoinCommand {
-    uint16_t command;
-    uint32_t flag;
-    std::string data;
-  };
-  std::unique_ptr<std::deque<JoinCommand>> game_join_command_queue;
-
-  // Character / game data
-  struct PendingItemTrade {
-    uint8_t other_client_id;
-    bool confirmed; // true if client has sent a D2 command
-    std::vector<ItemData> items;
-  };
-  struct PendingCardTrade {
-    uint8_t other_client_id;
-    bool confirmed; // true if client has sent an EE D2 command
-    std::vector<std::pair<uint32_t, uint32_t>> card_to_count;
-  };
-  bool should_update_play_time;
-  std::unordered_set<uint32_t> blocked_senders;
-  std::unique_ptr<PlayerDispDataDCPCV3> v1_v2_last_reported_disp;
-  // These are null unless the client is within the trade sequence (D0-D4 or EE commands)
-  std::unique_ptr<PendingItemTrade> pending_item_trade;
-  std::unique_ptr<PendingCardTrade> pending_card_trade;
-  std::shared_ptr<Episode3::PlayerConfig> ep3_config; // Null for non-Ep3
-  int8_t bb_character_index;
-  ItemData bb_identify_result;
-  std::array<std::vector<ItemData>, 3> bb_shop_contents;
-
-  // Miscellaneous (used by chat commands)
-  uint32_t next_exp_value; // next EXP value to give
-  G_SwitchStateChanged_6x05 last_switch_enabled_command;
-  bool can_chat;
-  struct PendingCharacterExport {
-    std::shared_ptr<const License> license;
-    ssize_t character_index = -1;
-    bool is_bb_conversion = false;
-  };
-  std::unique_ptr<PendingCharacterExport> pending_character_export;
-  std::deque<std::function<void(uint32_t, uint32_t)>> function_call_response_queue;
-
-  // File loading state
-  uint32_t dol_base_addr;
-  std::shared_ptr<DOLFileIndex::File> loading_dol_file;
-  std::unordered_map<std::string, std::shared_ptr<const std::string>> sending_files;
-
-  Client(
-      std::shared_ptr<Server> server,
-      struct bufferevent* bev,
-      Version version,
-      ServerBehavior server_behavior);
-  ~Client();
-
-  void reschedule_save_game_data_event();
-  void reschedule_ping_and_timeout_events();
-
-  inline Version version() const {
-    return this->channel.version;
-  }
-  inline uint8_t language() const {
-    return this->channel.language;
-  }
-
-  void set_license(std::shared_ptr<License> l);
-  void convert_license_to_temporary_if_nte();
-
-  void sync_config();
-
-  std::shared_ptr<ServerState> require_server_state() const;
-  std::shared_ptr<Lobby> require_lobby() const;
-
-  std::shared_ptr<const TeamIndex::Team> team() const;
-
-  bool can_see_quest(std::shared_ptr<const Quest> q, uint8_t event, uint8_t difficulty, size_t num_players) const;
-  bool can_play_quest(std::shared_ptr<const Quest> q, uint8_t event, uint8_t difficulty, size_t num_players) const;
-
-  static void dispatch_save_game_data(evutil_socket_t, short, void* ctx);
-  void save_game_data();
-  static void dispatch_send_ping(evutil_socket_t, short, void* ctx);
-  void send_ping();
-  static void dispatch_idle_timeout(evutil_socket_t, short, void* ctx);
-  void idle_timeout();
-
-  void suspend_timeouts();
-
-  const std::string& get_bb_username() const;
-  void set_bb_username(const std::string& bb_username);
-
-  void create_battle_overlay(std::shared_ptr<const BattleRules> rules, std::shared_ptr<const LevelTable> level_table);
-  void create_challenge_overlay(Version version, size_t template_index, std::shared_ptr<const LevelTable> level_table);
-  inline void delete_overlay() {
-    this->overlay_character_data.reset();
-  }
-  inline bool has_overlay() const {
-    return this->overlay_character_data.get() != nullptr;
-  }
-
-  void import_blocked_senders(const parray<le_uint32_t, 30>& blocked_senders);
-
-  std::shared_ptr<PSOBBBaseSystemFile> system_file(bool allow_load = true);
-  std::shared_ptr<PSOBBCharacterFile> character(bool allow_load = true, bool allow_overlay = true);
-  std::shared_ptr<PSOBBGuildCardFile> guild_card_file(bool allow_load = true);
-  std::shared_ptr<const PSOBBBaseSystemFile> system_file(bool allow_load = true) const;
-  std::shared_ptr<const PSOBBCharacterFile> character(bool allow_load = true, bool allow_overlay = true) const;
-  std::shared_ptr<const PSOBBGuildCardFile> guild_card_file(bool allow_load = true) const;
-
-  void create_character_file(
-      uint32_t guild_card_number,
-      uint8_t language,
-      const PlayerDispDataBBPreview& preview,
-      std::shared_ptr<const LevelTable> level_table);
-
-  std::string system_filename() const;
-  static std::string character_filename(const std::string& bb_username, int8_t index);
-  static std::string backup_character_filename(uint32_t serial_number, size_t index);
-  std::string character_filename(int8_t index = -1) const;
-  std::string guild_card_filename() const;
-  std::string shared_bank_filename() const;
-
-  std::string legacy_player_filename() const;
-  std::string legacy_account_filename() const;
-
-  void save_all();
-  void save_system_file() const;
-  static void save_character_file(
-      const std::string& filename,
-      std::shared_ptr<const PSOBBBaseSystemFile> sys,
-      std::shared_ptr<const PSOBBCharacterFile> character);
-  // Note: This function is not const because it updates the player's play time.
-  void save_character_file();
-  void save_guild_card_file() const;
-
-  void load_backup_character(uint32_t serial_number, size_t index);
-  void save_and_unload_character();
-
-  PlayerBank& current_bank();
-  std::shared_ptr<PSOBBCharacterFile> current_bank_character();
-  bool use_shared_bank(); // Returns true if the bank exists; false if it was created
-  void use_character_bank(int8_t bb_character_index);
-  void use_default_bank();
-
-  void print_inventory(FILE* stream) const;
-  void print_bank(FILE* stream) const;
-
-private:
-  // The overlay character data is used in battle and challenge modes, when
-  // character data is temporarily replaced in-game. In other play modes and in
-  // lobbies, overlay_character_data is null.
-  std::shared_ptr<PSOBBBaseSystemFile> system_data;
-  std::shared_ptr<PSOBBCharacterFile> overlay_character_data;
-  std::shared_ptr<PSOBBCharacterFile> character_data;
-  std::shared_ptr<PSOBBGuildCardFile> guild_card_data;
-  std::shared_ptr<PlayerBank> external_bank;
-  std::shared_ptr<PSOBBCharacterFile> external_bank_character;
-  int8_t external_bank_character_index;
-  uint64_t last_play_time_update;
-
-  void save_and_clear_external_bank();
-
-  void load_all_files();
-};
+#pragma once
+
+#include <netinet/in.h>
+
+#include <memory>
+#include <stdexcept>
+
+#include "Account.hh"
+#include "Channel.hh"
+#include "CommandFormats.hh"
+#include "Episode3/BattleRecord.hh"
+#include "Episode3/Tournament.hh"
+#include "FileContentsCache.hh"
+#include "FunctionCompiler.hh"
+#include "PSOEncryption.hh"
+#include "PSOProtocol.hh"
+#include "PatchFileIndex.hh"
+#include "Quest.hh"
+#include "QuestScript.hh"
+#include "TeamIndex.hh"
+#include "Text.hh"
+
+extern const uint64_t CLIENT_CONFIG_MAGIC;
+
+class Server;
+struct Lobby;
+class Parsed6x70Data;
+
+class Client : public std::enable_shared_from_this<Client> {
+public:
+  enum class Flag : uint64_t {
+    // clang-format off
+
+    // This mask specifies which flags are sent to the client
+    // TODO: It'd be nice to use a pattern here (e.g. all server-side flags are
+    // in the high bits) but that would require re-recording or manually
+    // rewriting all the tests
+    CLIENT_SIDE_MASK                           = 0xE73CFFFF7C0BFFFB,
+
+    // Version-related flags
+    CHECKED_FOR_DC_V1_PROTOTYPE                = 0x0000000000000002,
+    NO_D6_AFTER_LOBBY                          = 0x0000000000000100,
+    NO_D6                                      = 0x0000000000000200,
+    FORCE_ENGLISH_LANGUAGE_BB                  = 0x0000000000000400,
+
+    // Flags describing the behavior for send_function_call
+    HAS_SEND_FUNCTION_CALL                     = 0x0000000000001000,
+    ENCRYPTED_SEND_FUNCTION_CALL               = 0x0000000000002000,
+    SEND_FUNCTION_CALL_CHECKSUM_ONLY           = 0x0000000000004000,
+    SEND_FUNCTION_CALL_NO_CACHE_PATCH          = 0x0000000000008000,
+    CAN_RECEIVE_ENABLE_B2_QUEST                = 0x0000000000020000,
+    AWAITING_ENABLE_B2_QUEST                   = 0x0000000000040000, // Server-side only
+
+    // State flags
+    LOADING                                    = 0x0000000000100000, // Server-side only
+    LOADING_QUEST                              = 0x0000000000200000, // Server-side only
+    LOADING_RUNNING_JOINABLE_QUEST             = 0x0000000000400000, // Server-side only
+    LOADING_TOURNAMENT                         = 0x0000000000800000, // Server-side only
+    IN_INFORMATION_MENU                        = 0x0000000001000000, // Server-side only
+    AT_WELCOME_MESSAGE                         = 0x0000000002000000, // Server-side only
+    SAVE_ENABLED                               = 0x0000000004000000,
+    HAS_EP3_CARD_DEFS                          = 0x0000000008000000,
+    HAS_EP3_MEDIA_UPDATES                      = 0x0000000010000000,
+    USE_OVERRIDE_RANDOM_SEED                   = 0x0000000020000000,
+    HAS_GUILD_CARD_NUMBER                      = 0x0000000040000000,
+    HAS_AUTO_PATCHES                           = 0x0000004000000000,
+    AT_BANK_COUNTER                            = 0x0000000080000000, // Server-side only
+    SHOULD_SEND_ARTIFICIAL_ITEM_STATE          = 0x0001000000000000, // Server-side only
+    SHOULD_SEND_ARTIFICIAL_ENEMY_AND_SET_STATE = 0x0040000000000000, // Server-side only
+    SHOULD_SEND_ARTIFICIAL_OBJECT_STATE        = 0x0080000000000000, // Server-side only
+    SHOULD_SEND_ARTIFICIAL_FLAG_STATE          = 0x0002000000000000, // Server-side only
+    SHOULD_SEND_ARTIFICIAL_PLAYER_STATES       = 0x0200000000000000, // Server-side only
+    SHOULD_SEND_ENABLE_SAVE                    = 0x0004000000000000,
+    SWITCH_ASSIST_ENABLED                      = 0x0000000100000000,
+    IS_CLIENT_CUSTOMIZATION                    = 0x0100000000000000,
+    EP3_ALLOW_6xBC                             = 0x1000000000000000, // Server-side only
+
+    // Cheat mode and option flags
+    INFINITE_HP_ENABLED                        = 0x0000000200000000,
+    INFINITE_TP_ENABLED                        = 0x0000000400000000,
+    DEBUG_ENABLED                              = 0x0000000800000000,
+    ITEM_DROP_NOTIFICATIONS_1                  = 0x0010000000000000,
+    ITEM_DROP_NOTIFICATIONS_2                  = 0x0020000000000000,
+    FORCE_BATTLE_MODE_GAME                     = 0x0800000000000000, // Server-side only
+
+    // Proxy option flags
+    PROXY_SAVE_FILES                           = 0x0000001000000000,
+    PROXY_CHAT_COMMANDS_ENABLED                = 0x0000002000000000,
+    PROXY_PLAYER_NOTIFICATIONS_ENABLED         = 0x0000008000000000,
+    PROXY_SUPPRESS_CLIENT_PINGS                = 0x0000010000000000,
+    PROXY_SUPPRESS_REMOTE_LOGIN                = 0x0000020000000000,
+    PROXY_ZERO_REMOTE_GUILD_CARD               = 0x0000040000000000,
+    PROXY_EP3_INFINITE_MESETA_ENABLED          = 0x0000080000000000,
+    PROXY_EP3_INFINITE_TIME_ENABLED            = 0x0000100000000000,
+    PROXY_RED_NAME_ENABLED                     = 0x0000200000000000,
+    PROXY_BLANK_NAME_ENABLED                   = 0x0000400000000000,
+    PROXY_BLOCK_FUNCTION_CALLS                 = 0x0000800000000000,
+    PROXY_EP3_UNMASK_WHISPERS                  = 0x0008000000000000,
+    PROXY_VIRTUAL_CLIENT                       = 0x0400000000000000,
+    // clang-format on
+  };
+  enum class ItemDropNotificationMode {
+    NOTHING = 0,
+    RARES_ONLY = 1,
+    ALL_ITEMS = 2,
+    ALL_ITEMS_INCLUDING_MESETA = 3,
+  };
+
+  static constexpr uint64_t DEFAULT_FLAGS = static_cast<uint64_t>(Flag::PROXY_CHAT_COMMANDS_ENABLED);
+
+  struct Config {
+    uint64_t enabled_flags = DEFAULT_FLAGS; // Client::Flag enum
+    uint32_t specific_version = 0;
+    int32_t override_random_seed = 0;
+    uint8_t override_section_id = 0xFF; // FF = no override
+    uint8_t override_lobby_event = 0xFF; // FF = no override
+    uint8_t override_lobby_number = 0x80; // 80 = no override
+    uint32_t proxy_destination_address = 0;
+    uint16_t proxy_destination_port = 0;
+
+    Config() = default;
+
+    bool operator==(const Config& other) const = default;
+    bool operator!=(const Config& other) const = default;
+
+    bool should_update_vs(const Config& other) const;
+
+    [[nodiscard]] static inline bool check_flag(uint64_t enabled_flags, Flag flag) {
+      return !!(enabled_flags & static_cast<uint64_t>(flag));
+    }
+
+    [[nodiscard]] inline bool check_flag(Flag flag) const {
+      return this->check_flag(this->enabled_flags, flag);
+    }
+    inline void set_flag(Flag flag) {
+      this->enabled_flags |= static_cast<uint64_t>(flag);
+    }
+    inline void clear_flag(Flag flag) {
+      this->enabled_flags &= (~static_cast<uint64_t>(flag));
+    }
+    inline void toggle_flag(Flag flag) {
+      this->enabled_flags ^= static_cast<uint64_t>(flag);
+    }
+
+    void set_flags_for_version(Version version, int64_t sub_version);
+
+    ItemDropNotificationMode get_drop_notification_mode() const;
+    void set_drop_notification_mode(ItemDropNotificationMode new_mode);
+
+    template <size_t Bytes>
+    void parse_from(const parray<uint8_t, Bytes>& data) {
+      phosg::StringReader r(data.data(), data.size());
+      if (r.get_u32l() != CLIENT_CONFIG_MAGIC) {
+        throw std::invalid_argument("config signature is incorrect");
+      }
+      this->specific_version = r.get_u32l();
+      this->enabled_flags = r.get_u64l();
+      this->override_random_seed = r.get_u32l();
+      this->proxy_destination_address = r.get_u32b();
+      this->proxy_destination_port = r.get_u16l();
+      this->override_section_id = r.get_u8();
+      this->override_lobby_event = r.get_u8();
+      this->override_lobby_number = r.get_u8();
+    }
+
+    template <size_t Bytes>
+    void serialize_into(parray<uint8_t, Bytes>& data) const {
+      phosg::StringWriter w;
+      w.put_u32l(CLIENT_CONFIG_MAGIC);
+      w.put_u32l(this->specific_version);
+      w.put_u64l(this->enabled_flags & static_cast<uint64_t>(Flag::CLIENT_SIDE_MASK));
+      w.put_u32l(this->override_random_seed);
+      w.put_u32b(this->proxy_destination_address);
+      w.put_u16l(this->proxy_destination_port);
+      w.put_u8(this->override_section_id);
+      w.put_u8(this->override_lobby_event);
+      w.put_u8(this->override_lobby_number);
+
+      const auto& s = w.str();
+      for (size_t z = 0; z < s.size(); z++) {
+        data[z] = s[z];
+      }
+      data.clear_after(s.size(), 0xFF);
+    }
+  };
+
+  std::weak_ptr<Server> server;
+  uint64_t id;
+  phosg::PrefixedLogger log;
+
+  std::shared_ptr<Login> login;
+
+  // Network
+  Channel channel;
+  struct sockaddr_storage next_connection_addr;
+  ServerBehavior server_behavior;
+  bool should_disconnect;
+  bool should_send_to_lobby_server;
+  bool should_send_to_proxy_server;
+  std::unordered_map<std::string, std::function<void()>> disconnect_hooks;
+  std::shared_ptr<XBNetworkLocation> xb_netloc;
+  parray<le_uint32_t, 3> xb_9E_unknown_a1a;
+  uint8_t bb_connection_phase;
+  uint64_t ping_start_time;
+
+  // Lobby/positioning
+  Config config;
+  Config synced_config;
+  std::unique_ptr<parray<le_uint32_t, 0x20>> override_variations;
+  int32_t sub_version;
+  float x;
+  float z;
+  uint32_t floor;
+  std::weak_ptr<Lobby> lobby;
+  uint8_t lobby_client_id;
+  uint8_t lobby_arrow_color;
+  int64_t preferred_lobby_id; // <0 = no preference
+
+  std::unique_ptr<struct event, void (*)(struct event*)> save_game_data_event;
+  std::unique_ptr<struct event, void (*)(struct event*)> send_ping_event;
+  std::unique_ptr<struct event, void (*)(struct event*)> idle_timeout_event;
+  int16_t card_battle_table_number;
+  uint16_t card_battle_table_seat_number;
+  uint16_t card_battle_table_seat_state;
+  std::weak_ptr<Episode3::Tournament::Team> ep3_tournament_team;
+  std::shared_ptr<const Episode3::BattleRecord> ep3_prev_battle_record;
+  std::shared_ptr<const Menu> last_menu_sent;
+  uint32_t last_game_info_requested;
+  struct JoinCommand {
+    uint16_t command;
+    uint32_t flag;
+    std::string data;
+  };
+  std::unique_ptr<std::deque<JoinCommand>> game_join_command_queue;
+
+  // Character / game data
+  struct PendingItemTrade {
+    uint8_t other_client_id;
+    bool confirmed; // true if client has sent a D2 command
+    std::vector<ItemData> items;
+  };
+  struct PendingCardTrade {
+    uint8_t other_client_id;
+    bool confirmed; // true if client has sent an EE D2 command
+    std::vector<std::pair<uint32_t, uint32_t>> card_to_count;
+  };
+  bool should_update_play_time;
+  std::unordered_set<uint32_t> blocked_senders;
+  std::unique_ptr<PlayerDispDataDCPCV3> v1_v2_last_reported_disp;
+  std::shared_ptr<Parsed6x70Data> last_reported_6x70;
+  // These are null unless the client is within the trade sequence (D0-D4 or EE commands)
+  std::unique_ptr<PendingItemTrade> pending_item_trade;
+  std::unique_ptr<PendingCardTrade> pending_card_trade;
+  uint32_t telepipe_lobby_id;
+  G_SetTelepipeState_6x68 telepipe_state;
+  std::shared_ptr<Episode3::PlayerConfig> ep3_config; // Null for non-Ep3
+  int8_t bb_character_index;
+  ItemData bb_identify_result;
+  std::array<std::vector<ItemData>, 3> bb_shop_contents;
+
+  // Miscellaneous (used by chat commands)
+  uint32_t next_exp_value; // next EXP value to give
+  bool can_chat;
+  struct PendingCharacterExport {
+    std::shared_ptr<const Account> dest_account;
+    ssize_t character_index = -1;
+    std::shared_ptr<const BBLicense> dest_bb_license; // Only used for $bbchar; null for $savechar
+  };
+  std::unique_ptr<PendingCharacterExport> pending_character_export;
+  std::deque<std::function<void(uint32_t, uint32_t)>> function_call_response_queue;
+
+  // File loading state
+  uint32_t dol_base_addr;
+  std::shared_ptr<DOLFileIndex::File> loading_dol_file;
+  std::unordered_map<std::string, std::shared_ptr<const std::string>> sending_files;
+
+  Client(
+      std::shared_ptr<Server> server,
+      struct bufferevent* bev,
+      uint64_t virtual_network_id,
+      Version version,
+      ServerBehavior server_behavior);
+  ~Client();
+
+  void update_channel_name();
+
+  void reschedule_save_game_data_event();
+  void reschedule_ping_and_timeout_events();
+
+  inline Version version() const {
+    return this->channel.version;
+  }
+  inline uint8_t language() const {
+    return this->channel.language;
+  }
+
+  void convert_account_to_temporary_if_nte();
+
+  void sync_config();
+
+  std::shared_ptr<ServerState> require_server_state() const;
+  std::shared_ptr<Lobby> require_lobby() const;
+
+  std::shared_ptr<const TeamIndex::Team> team() const;
+
+  bool evaluate_quest_availability_expression(
+      std::shared_ptr<const IntegralExpression> expr,
+      std::shared_ptr<const Lobby> game,
+      uint8_t event,
+      uint8_t difficulty,
+      size_t num_players,
+      bool v1_present) const;
+  bool can_see_quest(
+      std::shared_ptr<const Quest> q,
+      std::shared_ptr<const Lobby> game,
+      uint8_t event,
+      uint8_t difficulty,
+      size_t num_players,
+      bool v1_present) const;
+  bool can_play_quest(
+      std::shared_ptr<const Quest> q,
+      std::shared_ptr<const Lobby> game,
+      uint8_t event,
+      uint8_t difficulty,
+      size_t num_players,
+      bool v1_present) const;
+
+  bool can_use_chat_commands() const;
+
+  static void dispatch_save_game_data(evutil_socket_t, short, void* ctx);
+  void save_game_data();
+  static void dispatch_send_ping(evutil_socket_t, short, void* ctx);
+  void send_ping();
+  static void dispatch_idle_timeout(evutil_socket_t, short, void* ctx);
+  void idle_timeout();
+
+  void suspend_timeouts();
+
+  const std::string& get_bb_username() const;
+  void set_bb_username(const std::string& bb_username);
+
+  void create_battle_overlay(std::shared_ptr<const BattleRules> rules, std::shared_ptr<const LevelTable> level_table);
+  void create_challenge_overlay(Version version, size_t template_index, std::shared_ptr<const LevelTable> level_table);
+  inline void delete_overlay() {
+    this->overlay_character_data.reset();
+  }
+  inline bool has_overlay() const {
+    return this->overlay_character_data.get() != nullptr;
+  }
+
+  void import_blocked_senders(const parray<le_uint32_t, 30>& blocked_senders);
+
+  std::shared_ptr<PSOBBBaseSystemFile> system_file(bool allow_load = true);
+  std::shared_ptr<PSOBBCharacterFile> character(bool allow_load = true, bool allow_overlay = true);
+  std::shared_ptr<PSOBBGuildCardFile> guild_card_file(bool allow_load = true);
+  std::shared_ptr<const PSOBBBaseSystemFile> system_file(bool allow_load = true) const;
+  std::shared_ptr<const PSOBBCharacterFile> character(bool allow_load = true, bool allow_overlay = true) const;
+  std::shared_ptr<const PSOBBGuildCardFile> guild_card_file(bool allow_load = true) const;
+
+  void create_character_file(
+      uint32_t guild_card_number,
+      uint8_t language,
+      const PlayerDispDataBBPreview& preview,
+      std::shared_ptr<const LevelTable> level_table);
+
+  std::string system_filename() const;
+  static std::string character_filename(const std::string& bb_username, int8_t index);
+  static std::string backup_character_filename(uint32_t account_id, size_t index, bool is_ep3);
+  std::string character_filename(int8_t index = -1) const;
+  std::string guild_card_filename() const;
+  std::string shared_bank_filename() const;
+
+  std::string legacy_player_filename() const;
+  std::string legacy_account_filename() const;
+
+  void save_all();
+  void save_system_file() const;
+  static void save_character_file(
+      const std::string& filename,
+      std::shared_ptr<const PSOBBBaseSystemFile> sys,
+      std::shared_ptr<const PSOBBCharacterFile> character);
+  static void save_ep3_character_file(
+      const std::string& filename,
+      const PSOGCEp3CharacterFile::Character& character);
+  // Note: This function is not const because it updates the player's play time.
+  void save_character_file();
+  void save_guild_card_file() const;
+
+  void load_backup_character(uint32_t account_id, size_t index);
+  std::shared_ptr<PSOGCEp3CharacterFile::Character> load_ep3_backup_character(uint32_t account_id, size_t index);
+  void save_and_unload_character();
+
+  PlayerBank200& current_bank();
+  const PlayerBank200& current_bank() const;
+  std::shared_ptr<PSOBBCharacterFile> current_bank_character();
+  bool use_shared_bank(); // Returns true if the bank exists; false if it was created
+  void use_character_bank(int8_t bb_character_index);
+  void use_default_bank();
+
+  void print_inventory(FILE* stream) const;
+  void print_bank(FILE* stream) const;
+
+private:
+  // The overlay character data is used in battle and challenge modes, when
+  // character data is temporarily replaced in-game. In other play modes and in
+  // lobbies, overlay_character_data is null.
+  std::shared_ptr<PSOBBBaseSystemFile> system_data;
+  std::shared_ptr<PSOBBCharacterFile> overlay_character_data;
+  std::shared_ptr<PSOBBCharacterFile> character_data;
+  std::shared_ptr<PSOBBGuildCardFile> guild_card_data;
+  std::shared_ptr<PlayerBank200> external_bank;
+  std::shared_ptr<PSOBBCharacterFile> external_bank_character;
+  int8_t external_bank_character_index;
+  uint64_t last_play_time_update;
+
+  void save_and_clear_external_bank();
+
+  void load_all_files();
+  void update_character_data_after_load(std::shared_ptr<PSOBBCharacterFile> character_data);
+};

src/CommonItemSet.cc

@@ -1,12 +1,431 @@
 #include "CommonItemSet.hh"
 
 #include "AFSArchive.hh"
+#include "EnemyType.hh"
 #include "GSLArchive.hh"
 #include "StaticGameData.hh"
+#include "Types.hh"
 
 using namespace std;
 
-CommonItemSet::Table::Table(const StringReader& r, bool is_big_endian, bool is_v3) {
+template <typename IntT, size_t Count>
+phosg::JSON to_json(const parray<IntT, Count>& v) {
+  auto ret = phosg::JSON::list();
+  for (size_t z = 0; z < Count; z++) {
+    ret.emplace_back(v[z]);
+  }
+  return ret;
+}
+
+template <typename IntT, size_t Count>
+void from_json_into(const phosg::JSON& json, parray<IntT, Count>& ret) {
+  if (json.size() != Count) {
+    throw runtime_error("incorrect array length");
+  }
+  for (size_t z = 0; z < Count; z++) {
+    ret[z] = json.at(z).as_int();
+  }
+}
+
+template <typename IntT, size_t Count>
+phosg::JSON to_json(const parray<CommonItemSet::Table::Range<IntT>, Count>& v) {
+  auto ret = phosg::JSON::list();
+  for (size_t z = 0; z < Count; z++) {
+    ret.emplace_back(to_json(v[z]));
+  }
+  return ret;
+}
+
+template <typename IntT, size_t Count>
+void from_json_into(const phosg::JSON& json, parray<CommonItemSet::Table::Range<IntT>, Count>& ret) {
+  if (json.size() != Count) {
+    throw runtime_error("incorrect array length");
+  }
+  for (size_t z = 0; z < Count; z++) {
+    from_json_into(json.at(z), ret[z]);
+  }
+}
+
+template <typename IntT>
+phosg::JSON to_json(const CommonItemSet::Table::Range<IntT>& v) {
+  if (v.min == v.max) {
+    return phosg::JSON(v.min);
+  } else {
+    return phosg::JSON::list({v.min, v.max});
+  }
+}
+
+template <typename IntT>
+void from_json_into(const phosg::JSON& json, CommonItemSet::Table::Range<IntT>& ret) {
+  if (json.is_int()) {
+    IntT v = json.as_int();
+    ret.min = v;
+    ret.max = v;
+  } else {
+    const auto& l = json.as_list();
+    if (l.size() != 2) {
+      throw runtime_error("incorrect range list length");
+    }
+    ret.min = l.at(0)->as_int();
+    ret.max = l.at(1)->as_int();
+  }
+}
+
+template <typename IntT, size_t Count1, size_t Count2>
+phosg::JSON to_json(const parray<parray<IntT, Count2>, Count1>& v) {
+  auto ret = phosg::JSON::list();
+  for (size_t z = 0; z < Count1; z++) {
+    ret.emplace_back(to_json(v[z]));
+  }
+  return ret;
+}
+
+template <typename IntT, size_t Count1, size_t Count2>
+void from_json_into(const phosg::JSON& json, parray<parray<IntT, Count2>, Count1>& ret) {
+  if (json.size() != Count1) {
+    throw runtime_error("incorrect array length");
+  }
+  for (size_t z = 0; z < Count1; z++) {
+    from_json_into(json.at(z), ret[z]);
+  }
+}
+
+template <typename IntT, size_t Count1, size_t Count2>
+void from_json_into(const phosg::JSON& json, parray<parray<CommonItemSet::Table::Range<IntT>, Count2>, Count1>& ret) {
+  if (json.size() != Count1) {
+    throw runtime_error("incorrect array length");
+  }
+  for (size_t z = 0; z < Count1; z++) {
+    from_json_into(json.at(z), ret[z]);
+  }
+}
+
+CommonItemSet::Table::Table(const phosg::JSON& json, Episode episode)
+    : episode(episode) {
+  from_json_into(json.at("BaseWeaponTypeProbTable"), this->base_weapon_type_prob_table);
+  from_json_into(json.at("SubtypeBaseTable"), this->subtype_base_table);
+  from_json_into(json.at("SubtypeAreaLengthTable"), this->subtype_area_length_table);
+  from_json_into(json.at("GrindProbTable"), this->grind_prob_table);
+  from_json_into(json.at("ArmorShieldTypeIndexProbTable"), this->armor_shield_type_index_prob_table);
+  from_json_into(json.at("ArmorSlotCountProbTable"), this->armor_slot_count_prob_table);
+  from_json_into(json.at("BoxMesetaRanges"), this->box_meseta_ranges);
+  this->has_rare_bonus_value_prob_table = json.at("HasRareBonusValueProbTable").as_bool();
+  from_json_into(json.at("BonusValueProbTable"), this->bonus_value_prob_table);
+  from_json_into(json.at("NonRareBonusProbSpec"), this->nonrare_bonus_prob_spec);
+  from_json_into(json.at("BonusTypeProbTable"), this->bonus_type_prob_table);
+  from_json_into(json.at("SpecialMult"), this->special_mult);
+  from_json_into(json.at("SpecialPercent"), this->special_percent);
+  from_json_into(json.at("ToolClassProbTable"), this->tool_class_prob_table);
+  from_json_into(json.at("TechniqueIndexProbTable"), this->technique_index_prob_table);
+  from_json_into(json.at("TechniqueLevelRanges"), this->technique_level_ranges);
+  this->armor_or_shield_type_bias = json.at("ArmorOrShieldTypeBias").as_int();
+  from_json_into(json.at("UnitMaxStarsTable"), this->unit_max_stars_table);
+  from_json_into(json.at("BoxItemClassProbTable"), this->box_item_class_prob_table);
+
+  const auto& enemy_meseta_ranges_json = json.at("EnemyMesetaRanges").as_dict();
+  const auto& enemy_type_drop_probs_json = json.at("EnemyTypeDropProbs").as_dict();
+  const auto& enemy_item_classes_json = json.at("EnemyItemClasses").as_dict();
+  for (size_t z = 0; z < 0x64; z++) {
+    static const array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
+    for (Episode episode : episodes) {
+      for (auto type : enemy_types_for_rare_table_index(episode, z)) {
+        string name = phosg::string_printf("%s:%s", abbreviation_for_episode(episode), phosg::name_for_enum(type));
+        from_json_into(*enemy_meseta_ranges_json.at(name), this->enemy_meseta_ranges[z]);
+        this->enemy_type_drop_probs[z] = enemy_type_drop_probs_json.at(name)->as_int();
+        this->enemy_item_classes[z] = enemy_item_classes_json.at(name)->as_int();
+      }
+    }
+  }
+}
+
+static const char* name_for_common_item_class(uint8_t item_class) {
+  switch (item_class) {
+    case 0x00:
+      return "WEAPON ";
+    case 0x01:
+      return "ARMOR  ";
+    case 0x02:
+      return "SHIELD ";
+    case 0x03:
+      return "UNIT   ";
+    case 0x04:
+      return "TOOL   ";
+    case 0x05:
+      return "MESETA ";
+    case 0x06:
+      return "NOTHING";
+    default:
+      return "UNKNOWN";
+  }
+}
+
+void CommonItemSet::Table::print(FILE* stream) const {
+  const auto& meseta_ranges = this->enemy_meseta_ranges;
+  const auto& drop_probs = this->enemy_type_drop_probs;
+  const auto& item_classes = this->enemy_item_classes;
+  fprintf(stream, "Enemy tables:\n");
+  fprintf(stream, "  ##   $LOW  $HIGH  DAR%%  ITEM        ENEMIES\n");
+  for (size_t z = 0; z < 0x64; z++) {
+    string enemies_str;
+    for (EnemyType enemy_type : enemy_types_for_rare_table_index(this->episode, z)) {
+      if (!enemies_str.empty()) {
+        enemies_str += ", ";
+      }
+      enemies_str += phosg::name_for_enum(enemy_type);
+    }
+    if (drop_probs[z]) {
+      fprintf(stream, "  %02zX  %5hu  %5hu  %3hhu%%  %02hX:%s  %s\n",
+          z, meseta_ranges[z].min, meseta_ranges[z].max, drop_probs[z], item_classes[z],
+          name_for_common_item_class(item_classes[z]), enemies_str.c_str());
+    } else {
+      fprintf(stream, "  %02zX  -----  -----    0%%  --          %s\n", z, enemies_str.c_str());
+    }
+  }
+
+  static const array<const char*, 12> base_weapon_type_names = {
+      "SABER   ",
+      "SWORD   ",
+      "DAGGER  ",
+      "PARTISAN",
+      "SLICER  ",
+      "HANDGUN ",
+      "RIFLE   ",
+      "MECHGUN ",
+      "SHOT    ",
+      "CANE    ",
+      "ROD     ",
+      "WAND    ",
+  };
+  fprintf(stream, "Base weapon config:\n");
+  fprintf(stream, "  TYPE         PROB  [SB  AL]  FLOORS\n");
+  for (size_t z = 0; z < 12; z++) {
+    uint8_t floor_to_class[10];
+    if (this->subtype_base_table[z] < 0) {
+      size_t start_floor = std::min<size_t>(-this->subtype_area_length_table[z], 10);
+      for (size_t x = 0; x < start_floor; x++) {
+        floor_to_class[x] = 0xFF;
+      }
+      for (size_t x = start_floor; x < 10; x++) {
+        floor_to_class[x] = (x - start_floor) / this->subtype_area_length_table[z];
+      }
+    } else {
+      for (size_t x = 0; x < 10; x++) {
+        floor_to_class[x] = this->subtype_base_table[z] + (x / this->subtype_area_length_table[z]);
+      }
+    }
+    fprintf(stream, "  %02zX:%s  %3hhu%%  [%02hhX  %02hhX]  %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX\n",
+        z, base_weapon_type_names[z], this->base_weapon_type_prob_table[z],
+        this->subtype_base_table[z], this->subtype_area_length_table[z],
+        floor_to_class[0], floor_to_class[1], floor_to_class[2], floor_to_class[3], floor_to_class[4],
+        floor_to_class[5], floor_to_class[6], floor_to_class[7], floor_to_class[8], floor_to_class[9]);
+  }
+
+  fprintf(stream, "Box configuration:\n");
+  fprintf(stream, "  AR   $LOW  $HIGH  WEP%%  ARM%%  SHD%%  UNI%%   TL%%  MST%%   NO%%\n");
+  for (size_t z = 0; z < 10; z++) {
+    fprintf(stream, "  %02zX  %5hu  %5hu  %3hhu%%  %3hhu%%  %3hhu%%  %3hhu%%  %3hhu%%  %3hhu%%  %3hhu%%\n",
+        z, this->box_meseta_ranges[z].min, this->box_meseta_ranges[z].max,
+        this->box_item_class_prob_table[0][z],
+        this->box_item_class_prob_table[1][z],
+        this->box_item_class_prob_table[2][z],
+        this->box_item_class_prob_table[3][z],
+        this->box_item_class_prob_table[4][z],
+        this->box_item_class_prob_table[5][z],
+        this->box_item_class_prob_table[6][z]);
+  }
+
+  fprintf(stream, "Weapon drops:\n");
+  fprintf(stream, "  Grinds:\n");
+  fprintf(stream, "    GD  AR0%%  AR1%%  AR2%%  AR3%%\n");
+  for (size_t z = 0; z < 9; z++) {
+    fprintf(stream, "    +%zu  %3hhd%%  %3hhd%%  %3hhd%%  %3hhd%%\n", z,
+        this->grind_prob_table[z][0], this->grind_prob_table[z][1],
+        this->grind_prob_table[z][2], this->grind_prob_table[z][3]);
+  }
+  fprintf(stream, "  Bonus value table:\n");
+  fprintf(stream, "    ID");
+  for (int8_t v = -10; v <= 100; v += 5) {
+    fprintf(stream, "  %5hhd%%", v);
+  }
+  fputc('\n', stream);
+  for (size_t z = 0; z < (this->has_rare_bonus_value_prob_table ? 6 : 5); z++) {
+    fprintf(stream, "    %02zX", z);
+    for (size_t x = 0; x < 0x17; x++) {
+      fprintf(stream, "  %5hu#", this->bonus_value_prob_table[x][z]);
+    }
+    fputc('\n', stream);
+  }
+  fprintf(stream, "  Area config tables:\n");
+  fprintf(stream, "    AR  BONUS  SP   NO%%  NTV%%   AB%%  MAC%%  DRK%%  HIT%%  SM  SPC%%\n");
+  for (size_t z = 0; z < 10; z++) {
+    fprintf(stream, "    %02zX  %02hhX %02hhX  %02hhX  %3hhu%%  %3hhu%%  %3hhu%%  %3hhu%%  %3hhu%%  %3hhu%%  %02hhX  %3hhu%%\n",
+        z, this->nonrare_bonus_prob_spec[0][z], this->nonrare_bonus_prob_spec[1][z], this->nonrare_bonus_prob_spec[2][z],
+        this->bonus_type_prob_table[0][z], this->bonus_type_prob_table[1][z], this->bonus_type_prob_table[2][z],
+        this->bonus_type_prob_table[3][z], this->bonus_type_prob_table[4][z], this->bonus_type_prob_table[5][z],
+        this->special_mult[z], this->special_percent[z]);
+  }
+
+  fprintf(stream, "  Tool class table:\n");
+  fprintf(stream, "    CS     A1     A2     A3     A4     A5     A6     A7     A8     A9    A10\n");
+  for (size_t tool_class = 0; tool_class < this->tool_class_prob_table.size(); tool_class++) {
+    fprintf(stream, "    %02zX", tool_class);
+    for (size_t area_norm = 0; area_norm < 10; area_norm++) {
+      fprintf(stream, "  %5hu", this->tool_class_prob_table[tool_class][area_norm]);
+    }
+    fputc('\n', stream);
+  }
+
+  static const array<const char*, 19> technique_names = {
+      "FOIE    ",
+      "GIFOIE  ",
+      "RAFOIE  ",
+      "BARTA   ",
+      "GIBARTA ",
+      "RABARTA ",
+      "ZONDE   ",
+      "GIZONDE ",
+      "RAZONDE ",
+      "GRANTS  ",
+      "DEBAND  ",
+      "JELLEN  ",
+      "ZALURE  ",
+      "SHIFTA  ",
+      "RYUKER  ",
+      "RESTA   ",
+      "ANTI    ",
+      "REVERSER",
+      "MEGID   ",
+  };
+
+  fprintf(stream, "  Technique table:\n");
+  fprintf(stream, "    TECH                   A1            A2            A3            A4            A5            A6            A7            A8            A9           A10\n");
+  for (size_t tech_num = 0; tech_num < this->technique_index_prob_table.size(); tech_num++) {
+    fprintf(stream, "    %02zX:%s", tech_num, technique_names[tech_num]);
+    for (size_t area_norm = 0; area_norm < 10; area_norm++) {
+      uint16_t prob = this->technique_index_prob_table[tech_num][area_norm];
+      if (prob) {
+        const auto& level_range = this->technique_level_ranges[tech_num][area_norm];
+        size_t min_level = level_range.min + 1;
+        size_t max_level = level_range.max + 1;
+        fprintf(stream, "  %5hu[%2zu-%2zu]", prob, min_level, max_level);
+      } else {
+        fprintf(stream, "      0[-----]");
+      }
+    }
+    fputc('\n', stream);
+  }
+
+  fprintf(stream, "  Armor/shield type bias: %hhu\n", this->armor_or_shield_type_bias);
+
+  fprintf(stream, "  Armor/shield type index table:\n");
+  fprintf(stream, "    TY  PROB\n");
+  for (size_t z = 0; z < 5; z++) {
+    fprintf(stream, "    %02zX  %3hhu%%\n", z, this->armor_shield_type_index_prob_table[z]);
+  }
+
+  fprintf(stream, "  Armor/shield slot count table:\n");
+  fprintf(stream, "    #S  PROB\n");
+  for (size_t z = 0; z < 5; z++) {
+    fprintf(stream, "    %02zX  %3hhu%%\n", z, this->armor_slot_count_prob_table[z]);
+  }
+
+  fprintf(stream, "  Unit maximum stars table:\n");
+  fprintf(stream, "    AR   #*\n");
+  for (size_t z = 0; z < 10; z++) {
+    fprintf(stream, "    %02zX  %3hhu\n", z, this->unit_max_stars_table[z]);
+  }
+}
+
+phosg::JSON CommonItemSet::Table::json() const {
+  phosg::JSON enemy_meseta_ranges_json = phosg::JSON::dict();
+  phosg::JSON enemy_type_drop_probs_json = phosg::JSON::dict();
+  phosg::JSON enemy_item_classes_json = phosg::JSON::dict();
+  for (size_t z = 0; z < 0x64; z++) {
+    static const array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
+    for (Episode episode : episodes) {
+      for (auto type : enemy_types_for_rare_table_index(episode, z)) {
+        string name = phosg::string_printf("%s:%s", abbreviation_for_episode(episode), phosg::name_for_enum(type));
+        enemy_meseta_ranges_json.emplace(name, to_json(this->enemy_meseta_ranges[z]));
+        enemy_type_drop_probs_json.emplace(name, this->enemy_type_drop_probs[z]);
+        enemy_item_classes_json.emplace(name, this->enemy_item_classes[z]);
+      }
+    }
+  }
+  return phosg::JSON::dict({
+      {"BaseWeaponTypeProbTable", to_json(this->base_weapon_type_prob_table)},
+      {"SubtypeBaseTable", to_json(this->subtype_base_table)},
+      {"SubtypeAreaLengthTable", to_json(this->subtype_area_length_table)},
+      {"GrindProbTable", to_json(this->grind_prob_table)},
+      {"ArmorShieldTypeIndexProbTable", to_json(this->armor_shield_type_index_prob_table)},
+      {"ArmorSlotCountProbTable", to_json(this->armor_slot_count_prob_table)},
+      {"EnemyMesetaRanges", std::move(enemy_meseta_ranges_json)},
+      {"EnemyTypeDropProbs", std::move(enemy_type_drop_probs_json)},
+      {"EnemyItemClasses", std::move(enemy_item_classes_json)},
+      {"BoxMesetaRanges", to_json(this->box_meseta_ranges)},
+      {"HasRareBonusValueProbTable", this->has_rare_bonus_value_prob_table},
+      {"BonusValueProbTable", to_json(this->bonus_value_prob_table)},
+      {"NonRareBonusProbSpec", to_json(this->nonrare_bonus_prob_spec)},
+      {"BonusTypeProbTable", to_json(this->bonus_type_prob_table)},
+      {"SpecialMult", to_json(this->special_mult)},
+      {"SpecialPercent", to_json(this->special_percent)},
+      {"ToolClassProbTable", to_json(this->tool_class_prob_table)},
+      {"TechniqueIndexProbTable", to_json(this->technique_index_prob_table)},
+      {"TechniqueLevelRanges", to_json(this->technique_level_ranges)},
+      {"ArmorOrShieldTypeBias", this->armor_or_shield_type_bias},
+      {"UnitMaxStarsTable", to_json(this->unit_max_stars_table)},
+      {"BoxItemClassProbTable", to_json(this->box_item_class_prob_table)},
+  });
+}
+
+phosg::JSON CommonItemSet::json() const {
+  auto modes_dict = phosg::JSON::dict();
+  static const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
+  for (const auto& mode : modes) {
+    auto episodes_dict = phosg::JSON::dict();
+    static const array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
+    for (const auto& episode : episodes) {
+      auto difficulty_dict = phosg::JSON::dict();
+      for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
+        auto section_id_dict = phosg::JSON::dict();
+        for (uint8_t section_id = 0; section_id < 10; section_id++) {
+          try {
+            auto table = this->get_table(episode, mode, difficulty, section_id);
+            section_id_dict.emplace(name_for_section_id(section_id), table->json());
+          } catch (const runtime_error&) {
+          }
+        }
+        difficulty_dict.emplace(token_name_for_difficulty(difficulty), std::move(section_id_dict));
+      }
+      episodes_dict.emplace(token_name_for_episode(episode), std::move(difficulty_dict));
+    }
+    modes_dict.emplace(name_for_mode(mode), std::move(episodes_dict));
+  }
+
+  return modes_dict;
+}
+
+void CommonItemSet::print(FILE* stream) const {
+  static const array<GameMode, 4> modes = {GameMode::NORMAL, GameMode::BATTLE, GameMode::CHALLENGE, GameMode::SOLO};
+  for (const auto& mode : modes) {
+    static const array<Episode, 3> episodes = {Episode::EP1, Episode::EP2, Episode::EP4};
+    for (const auto& episode : episodes) {
+      for (uint8_t difficulty = 0; difficulty < 4; difficulty++) {
+        for (uint8_t section_id = 0; section_id < 10; section_id++) {
+          try {
+            auto table = this->get_table(episode, mode, difficulty, section_id);
+            fprintf(stream, "============ %s %s %s %s\n",
+                name_for_mode(mode), name_for_episode(episode), name_for_difficulty(difficulty), name_for_section_id(section_id));
+            table->print(stream);
+          } catch (const runtime_error&) {
+          }
+        }
+      }
+    }
+  }
+}
+
+CommonItemSet::Table::Table(const phosg::StringReader& r, bool is_big_endian, bool is_v3, Episode episode)
+    : episode(episode) {
   if (is_big_endian) {
     this->parse_itempt_t<true>(r, is_v3);
   } else {
@@ -14,12 +433,9 @@ CommonItemSet::Table::Table(const StringReader& r, bool is_big_endian, bool is_v
   }
 }
 
-template <bool IsBigEndian>
-void CommonItemSet::Table::parse_itempt_t(const StringReader& r, bool is_v3) {
-  using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>::type;
-  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
-
-  const auto& offsets = r.pget<Offsets<IsBigEndian>>(r.pget<U32T>(r.size() - 0x10));
+template <bool BE>
+void CommonItemSet::Table::parse_itempt_t(const phosg::StringReader& r, bool is_v3) {
+  const auto& offsets = r.pget<OffsetsT<BE>>(r.pget<U32T<BE>>(r.size() - 0x10));
 
   this->base_weapon_type_prob_table = r.pget<parray<uint8_t, 0x0C>>(offsets.base_weapon_type_prob_table_offset);
   this->subtype_base_table = r.pget<parray<int8_t, 0x0C>>(offsets.subtype_base_table_offset);
@@ -27,14 +443,14 @@ void CommonItemSet::Table::parse_itempt_t(const StringReader& r, bool is_v3) {
   this->grind_prob_table = r.pget<parray<parray<uint8_t, 4>, 9>>(offsets.grind_prob_table_offset);
   this->armor_shield_type_index_prob_table = r.pget<parray<uint8_t, 0x05>>(offsets.armor_shield_type_index_prob_table_offset);
   this->armor_slot_count_prob_table = r.pget<parray<uint8_t, 0x05>>(offsets.armor_slot_count_prob_table_offset);
-  const auto& data = r.pget<parray<Range<U16T>, 0x64>>(offsets.enemy_meseta_ranges_offset);
+  const auto& data = r.pget<parray<Range<U16T<BE>>, 0x64>>(offsets.enemy_meseta_ranges_offset);
   for (size_t z = 0; z < data.size(); z++) {
     this->enemy_meseta_ranges[z] = Range<uint16_t>{data[z].min, data[z].max};
   }
   this->enemy_type_drop_probs = r.pget<parray<uint8_t, 0x64>>(offsets.enemy_type_drop_probs_offset);
   this->enemy_item_classes = r.pget<parray<uint8_t, 0x64>>(offsets.enemy_item_classes_offset);
   {
-    const auto& data = r.pget<parray<Range<U16T>, 0x0A>>(offsets.box_meseta_ranges_offset);
+    const auto& data = r.pget<parray<Range<U16T<BE>>, 0x0A>>(offsets.box_meseta_ranges_offset);
     for (size_t z = 0; z < data.size(); z++) {
       this->box_meseta_ranges[z] = Range<uint16_t>{data[z].min, data[z].max};
     }
@@ -48,7 +464,7 @@ void CommonItemSet::Table::parse_itempt_t(const StringReader& r, bool is_v3) {
       }
     }
   } else { // V3
-    const auto& data = r.pget<parray<parray<U16T, 6>, 0x17>>(offsets.bonus_value_prob_table_offset);
+    const auto& data = r.pget<parray<parray<U16T<BE>, 6>, 0x17>>(offsets.bonus_value_prob_table_offset);
     for (size_t z = 0; z < data.size(); z++) {
       for (size_t x = 0; x < data[z].size(); x++) {
         this->bonus_value_prob_table[z][x] = data[z][x];
@@ -60,7 +476,7 @@ void CommonItemSet::Table::parse_itempt_t(const StringReader& r, bool is_v3) {
   this->special_mult = r.pget<parray<uint8_t, 0x0A>>(offsets.special_mult_offset);
   this->special_percent = r.pget<parray<uint8_t, 0x0A>>(offsets.special_percent_offset);
   {
-    const auto& data = r.pget<parray<parray<U16T, 0x0A>, 0x1C>>(offsets.tool_class_prob_table_offset);
+    const auto& data = r.pget<parray<parray<U16T<BE>, 0x0A>, 0x1C>>(offsets.tool_class_prob_table_offset);
     for (size_t z = 0; z < data.size(); z++) {
       for (size_t x = 0; x < data[z].size(); x++) {
         this->tool_class_prob_table[z][x] = data[z][x];
@@ -74,41 +490,6 @@ void CommonItemSet::Table::parse_itempt_t(const StringReader& r, bool is_v3) {
   this->box_item_class_prob_table = r.pget<parray<parray<uint8_t, 10>, 7>>(offsets.box_item_class_prob_table_offset);
 }
 
-void CommonItemSet::Table::print_enemy_table(FILE* stream) const {
-  const auto& meseta_ranges = this->enemy_meseta_ranges;
-  const auto& drop_probs = this->enemy_type_drop_probs;
-  const auto& item_classes = this->enemy_item_classes;
-  // const parray<Range<uint16_t>, 0x64>& enemy_meseta_ranges() const;
-  //   const parray<uint8_t, 0x64>& enemy_type_drop_probs() const;
-  //   const parray<uint8_t, 0x64>& enemy_item_classes() const;
-  fprintf(stream, "##  $_LOW  $_HIGH  DAR  ITEM\n");
-  for (size_t z = 0; z < 0x64; z++) {
-    const char* item_class_name = "__UNKNOWN__";
-    switch (item_classes[z]) {
-      case 0x00:
-        item_class_name = "WEAPON";
-        break;
-      case 0x01:
-        item_class_name = "ARMOR";
-        break;
-      case 0x02:
-        item_class_name = "SHIELD";
-        break;
-      case 0x03:
-        item_class_name = "UNIT";
-        break;
-      case 0x04:
-        item_class_name = "TOOL";
-        break;
-      case 0x05:
-        item_class_name = "MESETA";
-        break;
-    }
-    fprintf(stream, "%02zX  %5hu   %5hu  %3hhu  %02hX (%s)\n",
-        z, meseta_ranges[z].min, meseta_ranges[z].max, drop_probs[z], item_classes[z], item_class_name);
-  }
-}
-
 uint16_t CommonItemSet::key_for_table(Episode episode, GameMode mode, uint8_t difficulty, uint8_t secid) {
   // Bits: -----EEEMMDDSSSS
   return (((static_cast<uint16_t>(episode) << 8) & 0x0700) |
@@ -122,7 +503,7 @@ shared_ptr<const CommonItemSet::Table> CommonItemSet::get_table(
   try {
     return this->tables.at(this->key_for_table(episode, mode, difficulty, secid));
   } catch (const out_of_range&) {
-    throw runtime_error(string_printf("common item table not available for episode=%s, mode=%s, difficulty=%hu, secid=%hu",
+    throw runtime_error(phosg::string_printf("common item table not available for episode=%s, mode=%s, difficulty=%hu, secid=%hu",
         name_for_episode(episode), name_for_mode(mode), difficulty, secid));
   }
 }
@@ -135,8 +516,8 @@ AFSV2CommonItemSet::AFSV2CommonItemSet(
   for (size_t difficulty = 0; difficulty < 4; difficulty++) {
     for (size_t section_id = 0; section_id < 10; section_id++) {
       auto entry = pt_afs.get(difficulty * 10 + section_id);
-      StringReader r(entry.first, entry.second);
-      auto table = make_shared<Table>(r, false, false);
+      phosg::StringReader r(entry.first, entry.second);
+      auto table = make_shared<Table>(r, false, false, Episode::EP1);
       this->tables.emplace(this->key_for_table(Episode::EP1, GameMode::NORMAL, difficulty, section_id), table);
       this->tables.emplace(this->key_for_table(Episode::EP1, GameMode::BATTLE, difficulty, section_id), table);
       this->tables.emplace(this->key_for_table(Episode::EP1, GameMode::SOLO, difficulty, section_id), table);
@@ -148,7 +529,7 @@ AFSV2CommonItemSet::AFSV2CommonItemSet(
   AFSArchive ct_afs(ct_afs_data);
   for (size_t difficulty = 0; difficulty < 4; difficulty++) {
     auto r = ct_afs.get_reader(difficulty * 10);
-    auto table = make_shared<Table>(r, false, false);
+    auto table = make_shared<Table>(r, false, false, Episode::EP1);
     for (size_t section_id = 0; section_id < 10; section_id++) {
       this->tables.emplace(this->key_for_table(Episode::EP1, GameMode::CHALLENGE, difficulty, section_id), table);
     }
@@ -173,7 +554,7 @@ GSLV3V4CommonItemSet::GSLV3V4CommonItemSet(std::shared_ptr<const std::string> gs
       default:
         throw runtime_error("invalid episode");
     }
-    return string_printf(
+    return phosg::string_printf(
         "ItemPT%s%s%c%1hhu.rel",
         is_challenge ? "c" : "",
         episode_token,
@@ -185,7 +566,7 @@ GSLV3V4CommonItemSet::GSLV3V4CommonItemSet(std::shared_ptr<const std::string> gs
   for (Episode episode : episodes) {
     for (size_t difficulty = 0; difficulty < 4; difficulty++) {
       for (size_t section_id = 0; section_id < 10; section_id++) {
-        StringReader r;
+        phosg::StringReader r;
         try {
           r = gsl.get_reader(filename_for_table(episode, difficulty, section_id, false));
         } catch (const exception&) {
@@ -200,7 +581,7 @@ GSLV3V4CommonItemSet::GSLV3V4CommonItemSet(std::shared_ptr<const std::string> gs
             throw;
           }
         }
-        auto table = make_shared<Table>(r, is_big_endian, true);
+        auto table = make_shared<Table>(r, is_big_endian, true, episode);
         this->tables.emplace(this->key_for_table(episode, GameMode::NORMAL, difficulty, section_id), table);
         this->tables.emplace(this->key_for_table(episode, GameMode::BATTLE, difficulty, section_id), table);
         this->tables.emplace(this->key_for_table(episode, GameMode::SOLO, difficulty, section_id), table);
@@ -210,7 +591,7 @@ GSLV3V4CommonItemSet::GSLV3V4CommonItemSet(std::shared_ptr<const std::string> gs
     if (episode != Episode::EP4) {
       for (size_t difficulty = 0; difficulty < 4; difficulty++) {
         auto r = gsl.get_reader(filename_for_table(episode, difficulty, 0, true));
-        auto table = make_shared<Table>(r, is_big_endian, true);
+        auto table = make_shared<Table>(r, is_big_endian, true, episode);
         for (size_t section_id = 0; section_id < 10; section_id++) {
           this->tables.emplace(this->key_for_table(episode, GameMode::CHALLENGE, difficulty, section_id), table);
         }
@@ -219,6 +600,33 @@ GSLV3V4CommonItemSet::GSLV3V4CommonItemSet(std::shared_ptr<const std::string> gs
   }
 }
 
+JSONCommonItemSet::JSONCommonItemSet(const phosg::JSON& json) {
+  for (const auto& mode_it : json.as_dict()) {
+    static const unordered_map<string, GameMode> mode_keys(
+        {{"Normal", GameMode::NORMAL}, {"Battle", GameMode::BATTLE}, {"Challenge", GameMode::CHALLENGE}, {"Solo", GameMode::SOLO}});
+    GameMode mode = mode_keys.at(mode_it.first);
+
+    for (const auto& episode_it : mode_it.second->as_dict()) {
+      static const unordered_map<string, Episode> episode_keys(
+          {{"Episode1", Episode::EP1}, {"Episode2", Episode::EP2}, {"Episode4", Episode::EP4}});
+      Episode episode = episode_keys.at(episode_it.first);
+
+      for (const auto& difficulty_it : episode_it.second->as_dict()) {
+        static const unordered_map<string, uint8_t> difficulty_keys(
+            {{"Normal", 0}, {"Hard", 1}, {"VeryHard", 2}, {"Ultimate", 3}});
+        uint8_t difficulty = difficulty_keys.at(difficulty_it.first);
+
+        for (const auto& section_id_it : difficulty_it.second->as_dict()) {
+          uint8_t section_id = section_id_for_name(section_id_it.first);
+          this->tables.emplace(
+              this->key_for_table(episode, mode, difficulty, section_id),
+              make_shared<Table>(*section_id_it.second, episode));
+        }
+      }
+    }
+  }
+}
+
 RELFileSet::RELFileSet(std::shared_ptr<const std::string> data)
     : data(data),
       r(*this->data) {}
@@ -381,7 +789,7 @@ const ProbabilityTable<uint8_t, 100>& TekkerAdjustmentSet::get_bonus_delta_prob_
 }
 
 int8_t TekkerAdjustmentSet::get_luck(uint32_t start_offset, uint8_t delta_index) const {
-  StringReader sub_r = r.sub(start_offset);
+  phosg::StringReader sub_r = r.sub(start_offset);
   while (!sub_r.eof()) {
     const auto& entry = sub_r.get<LuckTableEntry>();
     if (entry.delta_index == 0xFF) {

src/CommonItemSet.hh

@@ -2,25 +2,29 @@
 
 #include <array>
 #include <phosg/Encoding.hh>
+#include <phosg/JSON.hh>
 
 #include "GSLArchive.hh"
 #include "PSOEncryption.hh"
 #include "StaticGameData.hh"
 #include "Text.hh"
+#include "Types.hh"
 
 class CommonItemSet {
 public:
   class Table {
   public:
     Table() = delete;
-    Table(const StringReader& r, bool big_endian, bool is_v3);
+    Table(const phosg::JSON& json, Episode episode);
+    Table(const phosg::StringReader& r, bool big_endian, bool is_v3, Episode episode);
 
     template <typename IntT>
     struct Range {
       IntT min;
       IntT max;
-    } __attribute__((packed));
+    } __packed__;
 
+    Episode episode;
     parray<uint8_t, 0x0C> base_weapon_type_prob_table;
     parray<int8_t, 0x0C> subtype_base_table;
     parray<uint8_t, 0x0C> subtype_area_length_table;
@@ -44,17 +48,15 @@ public:
     parray<uint8_t, 0x0A> unit_max_stars_table;
     parray<parray<uint8_t, 10>, 7> box_item_class_prob_table;
 
-    void print_enemy_table(FILE* stream) const;
+    phosg::JSON json() const;
+    void print(FILE* stream) const;
 
   private:
-    template <bool IsBigEndian>
-    void parse_itempt_t(const StringReader& r, bool is_v3);
-
-    template <bool IsBigEndian>
-    struct Offsets {
-      using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>::type;
-      using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+    template <bool BE>
+    void parse_itempt_t(const phosg::StringReader& r, bool is_v3);
 
+    template <bool BE>
+    struct OffsetsT {
       // This data structure uses index probability tables in multiple places. An
       // index probability table is a table where each entry holds the probability
       // that that entry's index is used. For example, if the armor slot count
@@ -76,7 +78,7 @@ public:
       // The indexes in this table correspond to the non-rare weapon types 01
       // through 0C (Saber through Wand).
       // V2/V3: -> parray<uint8_t, 0x0C>
-      /* 00 */ U32T base_weapon_type_prob_table_offset;
+      /* 00 */ U32T<BE> base_weapon_type_prob_table_offset;
 
       // This table specifies the base subtype for each weapon type. Negative
       // values here mean that the weapon cannot be found in the first N areas (so
@@ -86,7 +88,7 @@ public:
       // of weapon that actually appears depends on this value and a value from
       // the following table.
       // V2/V3: -> parray<int8_t, 0x0C>
-      /* 04 */ U32T subtype_base_table_offset;
+      /* 04 */ U32T<BE> subtype_base_table_offset;
 
       // This table specifies how many areas each weapon subtype appears in. For
       // example, if Sword (subtype 02, which is index 1 in this table and the
@@ -95,7 +97,7 @@ public:
       // through Mine 1), and Gigush (the next sword subtype) can be found in Mine
       // 1 through Ruins 3.
       // V2/V3: -> parray<uint8_t, 0x0C>
-      /* 08 */ U32T subtype_area_length_table_offset;
+      /* 08 */ U32T<BE> subtype_area_length_table_offset;
 
       // This index probability table specifies how likely each possible grind
       // value is. The table is indexed as [grind][subtype_area_index], where the
@@ -110,28 +112,28 @@ public:
       //    ...
       //    C1 C2 C3 M1  // (Episode 1 area values from the example for reference)
       // V2/V3: -> parray<parray<uint8_t, 4>, 9>
-      /* 0C */ U32T grind_prob_table_offset;
+      /* 0C */ U32T<BE> grind_prob_table_offset;
 
       // TODO: Figure out exactly how this table is used. Anchor: 80106D34
       // V2/V3: -> parray<uint8_t, 0x05>
-      /* 10 */ U32T armor_shield_type_index_prob_table_offset;
+      /* 10 */ U32T<BE> armor_shield_type_index_prob_table_offset;
 
       // This index probability table specifies how common each possible slot
       // count is for armor drops.
       // V2/V3: -> parray<uint8_t, 0x05>
-      /* 14 */ U32T armor_slot_count_prob_table_offset;
+      /* 14 */ U32T<BE> armor_slot_count_prob_table_offset;
 
       // This array (indexed by enemy_type) specifies the range of meseta values
       // that each enemy can drop.
       // V2/V3: -> parray<Range<U16T>, 0x64>
-      /* 18 */ U32T enemy_meseta_ranges_offset;
+      /* 18 */ U32T<BE> enemy_meseta_ranges_offset;
 
       // Each byte in this table (indexed by enemy_type) represents the percent
       // chance that the enemy drops anything at all. (This check is done before
       // the rare drop check, so the chance of getting a rare item from an enemy
       // is essentially this probability multiplied by the rare drop rate.)
       // V2/V3: -> parray<uint8_t, 0x64>
-      /* 1C */ U32T enemy_type_drop_probs_offset;
+      /* 1C */ U32T<BE> enemy_type_drop_probs_offset;
 
       // Each byte in this table (indexed by enemy_type) represents the class of
       // item that the enemy can drop. The values are:
@@ -143,12 +145,12 @@ public:
       // 05 = meseta
       // Anything else = no item
       // V2/V3: -> parray<uint8_t, 0x64>
-      /* 20 */ U32T enemy_item_classes_offset;
+      /* 20 */ U32T<BE> enemy_item_classes_offset;
 
       // This table (indexed by area - 1) specifies the ranges of meseta values
       // that can drop from boxes.
       // V2/V3: -> parray<Range<U16T>, 0x0A>
-      /* 24 */ U32T box_meseta_ranges_offset;
+      /* 24 */ U32T<BE> box_meseta_ranges_offset;
 
       // This array specifies the chance that a rare weapon will have each
       // possible bonus value. This is indexed as [(bonus_value - 10 / 5)][spec],
@@ -158,7 +160,7 @@ public:
       // for rare items, spec is always 5.
       // V2: -> parray<parray<uint8_t, 5>, 0x17>
       // V3: -> parray<parray<U16T, 6>, 0x17>
-      /* 28 */ U32T bonus_value_prob_table_offset;
+      /* 28 */ U32T<BE> bonus_value_prob_table_offset;
 
       // This array specifies the value of spec to be used in the above lookup for
       // non-rare items. This is NOT an index probability table; this is a direct
@@ -174,7 +176,7 @@ public:
       // bonus; in all other areas except Ruins 3, they can have at most two
       // bonuses, and in Ruins 3, they can have up to three bonuses.
       // V2/V3: // -> parray<parray<uint8_t, 10>, 3>
-      /* 2C */ U32T nonrare_bonus_prob_spec_offset;
+      /* 2C */ U32T<BE> nonrare_bonus_prob_spec_offset;
 
       // This array specifies the chance that a weapon will have each bonus type.
       // The table is indexed as [bonus_type][area - 1] for non-rare items; for
@@ -189,37 +191,37 @@ public:
       //   [00 00 00 00 00 01 01 01 01 01] // Chance of getting Hit bonus
       //    F1 F2 C1 C2 C3 M1 M2 R1 R2 R3  // (Episode 1 areas, for reference)
       // V2/V3: -> parray<parray<uint8_t, 10>, 6>
-      /* 30 */ U32T bonus_type_prob_table_offset;
+      /* 30 */ U32T<BE> bonus_type_prob_table_offset;
 
       // This array (indexed by area - 1) specifies a multiplier of used in
       // special ability determination. It seems this uses the star values from
       // ItemPMT, but not yet clear exactly in what way.
       // TODO: Figure out exactly what this does. Anchor: 80106FEC
       // V2/V3: -> parray<uint8_t, 0x0A>
-      /* 34 */ U32T special_mult_offset;
+      /* 34 */ U32T<BE> special_mult_offset;
 
       // This array (indexed by area - 1) specifies the probability that any
       // non-rare weapon will have a special ability.
       // V2/V3: -> parray<uint8_t, 0x0A>
-      /* 38 */ U32T special_percent_offset;
+      /* 38 */ U32T<BE> special_percent_offset;
 
       // This index probability table is indexed by [tool_class][area - 1]. The
       // tool class refers to an entry in ItemPMT, which links it to the actual
       // item code.
       // V2/V3: -> parray<parray<U16T, 0x0A>, 0x1C>
-      /* 3C */ U32T tool_class_prob_table_offset;
+      /* 3C */ U32T<BE> tool_class_prob_table_offset;
 
       // This index probability table determines how likely each technique is to
       // appear. The table is indexed as [technique_num][area - 1].
       // V2/V3: -> parray<parray<uint8_t, 0x0A>, 0x13>
-      /* 40 */ U32T technique_index_prob_table_offset;
+      /* 40 */ U32T<BE> technique_index_prob_table_offset;
 
       // This table specifies the ranges for technique disk levels. The table is
       // indexed as [technique_num][area - 1]. If either min or max in the range
       // is 0xFF, or if max < min, technique disks are not dropped for that
       // technique and area pair.
       // V2/V3: -> parray<parray<Range<uint8_t>, 0x0A>, 0x13>
-      /* 44 */ U32T technique_level_ranges_offset;
+      /* 44 */ U32T<BE> technique_level_ranges_offset;
 
       /* 48 */ uint8_t armor_or_shield_type_bias;
       /* 49 */ parray<uint8_t, 3> unused1;
@@ -230,7 +232,7 @@ public:
       // game uniformly chooses a random number of stars in the acceptable
       // range, then uniformly chooses a random unit with that many stars.
       // V2/V3: -> parray<uint8_t, 0x0A>
-      /* 4C */ U32T unit_max_stars_offset;
+      /* 4C */ U32T<BE> unit_max_stars_offset;
 
       // This index probability table determines which type of items drop from
       // boxes. The table is indexed as [item_class][area - 1], with item_class
@@ -249,13 +251,19 @@ public:
       //   [16 16 11 11 11 11 11 0F 0C 0B] // Chances per area of an empty box
       //    F1 F2 C1 C2 C3 M1 M2 R1 R2 R3  // (Episode 1 areas, for reference)
       // V2/V3: -> parray<parray<uint8_t, 10>, 7>
-      /* 50 */ U32T box_item_class_prob_table_offset;
+      /* 50 */ U32T<BE> box_item_class_prob_table_offset;
 
       // There are several unused fields here.
-    } __attribute__((packed));
+    } __packed__;
+    using Offsets = OffsetsT<false>;
+    using OffsetsBE = OffsetsT<true>;
+    check_struct_size(Offsets, 0x54);
+    check_struct_size(OffsetsBE, 0x54);
   };
 
   std::shared_ptr<const Table> get_table(Episode episode, GameMode mode, uint8_t difficulty, uint8_t secid) const;
+  phosg::JSON json() const;
+  void print(FILE* stream) const;
 
 protected:
   CommonItemSet() = default;
@@ -275,6 +283,11 @@ public:
   GSLV3V4CommonItemSet(std::shared_ptr<const std::string> gsl_data, bool is_big_endian);
 };
 
+class JSONCommonItemSet : public CommonItemSet {
+public:
+  explicit JSONCommonItemSet(const phosg::JSON& json);
+};
+
 // Note: There are clearly better ways of doing this, but this implementation
 // closely follows what the original code in the client does.
 template <typename ItemT, size_t MaxCount>
@@ -298,22 +311,22 @@ struct ProbabilityTable {
     return this->items[--this->count];
   }
 
-  void shuffle(PSOLFGEncryption& random_crypt) {
+  void shuffle(std::shared_ptr<PSOLFGEncryption> opt_rand_crypt) {
     for (size_t z = 1; z < this->count; z++) {
-      size_t other_z = random_crypt.next() % (z + 1);
+      size_t other_z = random_from_optional_crypt(opt_rand_crypt) % (z + 1);
       ItemT t = this->items[z];
       this->items[z] = this->items[other_z];
       this->items[other_z] = t;
     }
   }
 
-  ItemT sample(PSOLFGEncryption& random_crypt) const {
+  ItemT sample(std::shared_ptr<PSOLFGEncryption> opt_rand_crypt) const {
     if (this->count == 0) {
       throw std::runtime_error("sample from empty probability table");
     } else if (this->count == 1) {
       return this->items[0];
     } else {
-      return this->items[random_crypt.next() % this->count];
+      return this->items[random_from_optional_crypt(opt_rand_crypt) % this->count];
     }
   }
 };
@@ -324,20 +337,22 @@ public:
   struct WeightTableEntry {
     ValueT value;
     WeightT weight;
-  } __attribute__((packed));
+  } __packed__;
 
   using WeightTableEntry8 = WeightTableEntry<uint8_t>;
   using WeightTableEntry32 = WeightTableEntry<be_uint32_t>;
+  check_struct_size(WeightTableEntry8, 2);
+  check_struct_size(WeightTableEntry32, 8);
 
 protected:
   std::shared_ptr<const std::string> data;
-  StringReader r;
+  phosg::StringReader r;
 
   struct TableSpec {
     be_uint32_t offset;
     uint8_t entries_per_table;
     parray<uint8_t, 3> unused;
-  } __attribute__((packed));
+  } __packed_ws__(TableSpec, 8);
 
   RELFileSet(std::shared_ptr<const std::string> data);
 
@@ -378,7 +393,7 @@ public:
     Mode mode;
     uint8_t player_level_divisor_or_min_level;
     uint8_t max_level;
-  } __attribute__((packed));
+  } __packed_ws__(TechDiskLevelEntry, 3);
 
   std::pair<const uint8_t*, size_t> get_common_recovery_table(size_t index) const;
   std::pair<const WeightTableEntry8*, size_t> get_rare_recovery_table(size_t index) const;
@@ -400,7 +415,7 @@ public:
   struct RangeTableEntry {
     be_uint32_t min;
     be_uint32_t max;
-  } __attribute__((packed));
+  } __packed_ws__(RangeTableEntry, 8);
 
   std::pair<const WeightTableEntry8*, size_t> get_weapon_type_table(size_t index) const;
   const parray<WeightTableEntry32, 6>* get_bonus_type_table(size_t which, size_t index) const;
@@ -419,7 +434,7 @@ private:
     be_uint32_t special_mode_table; // [[{u32 value, u32 weight}](3)](8)
     be_uint32_t standard_grind_range_table; // [{u32 min, u32 max}](6)
     be_uint32_t favored_grind_range_table; // [{u32 min, u32 max}](6)
-  } __attribute__((packed));
+  } __packed_ws__(Offsets, 0x20);
 
   const Offsets* offsets;
 };
@@ -446,17 +461,17 @@ private:
   int8_t get_luck(uint32_t start_offset, uint8_t delta_index) const;
 
   std::shared_ptr<const std::string> data;
-  StringReader r;
+  phosg::StringReader r;
 
   struct DeltaProbabilityEntry {
     uint8_t delta_index;
     uint8_t count_default;
     uint8_t count_favored;
-  } __attribute__((packed));
+  } __packed_ws__(DeltaProbabilityEntry, 3);
   struct LuckTableEntry {
     uint8_t delta_index;
     int8_t luck;
-  } __attribute__((packed));
+  } __packed_ws__(LuckTableEntry, 2);
 
   struct Offsets {
     // Each section ID's favored weapon class has different probabilities than
@@ -562,7 +577,7 @@ private:
     // In PSO V3, the bonus delta luck table is:
     //   +10 => +15, +5 => +8, 0 => 0, -5 => -8, -10 => -15
     be_uint32_t bonus_delta_luck_offset; // LuckTableEntry[...]; ending with FF FF
-  } __attribute__((packed));
+  } __packed_ws__(Offsets, 0x18);
 
   const Offsets* offsets;
 

src/Compression.hh

@@ -1,222 +1,226 @@
-#pragma once
-
-#include <stddef.h>
-
-#include <array>
-#include <deque>
-#include <functional>
-#include <phosg/Tools.hh>
-#include <string>
-
-#include "Text.hh"
-
-enum class CompressPhase {
-  INDEX = 0,
-  CONSTRUCT_PATHS,
-  BACKTRACE_OPTIMAL_PATH,
-  GENERATE_RESULT,
-};
-
-template <>
-const char* name_for_enum<CompressPhase>(CompressPhase v);
-
-typedef std::function<void(CompressPhase phase, size_t input_progress, size_t input_size, size_t output_size)> ProgressCallback;
-
-////////////////////////////////////////////////////////////////////////////////
-// PRS compression
-////////////////////////////////////////////////////////////////////////////////
-
-// Use this class if you need to compress from multiple input buffers, or need
-// to compress multiple chunks and don't want to copy their contents
-// unnecessarily. (For most common use cases, use prs_compress, below, instead.)
-// To use this class, instantiate it, then call .add() one or more times, then
-// call .close() and use the returned string as the compressed result.
-class PRSCompressor {
-public:
-  // compression_level specifies how aggressively to search for alternate paths:
-  //   -1: Don't perform any compression at all, but produce output that can be
-  //       understood by prs_decompress. The output will be about 9/8 the size
-  //       of the input.
-  //   0:  Greedily search for the longest backreference at every point. Don't
-  //       consider any alternate paths. Generally offers a good balance between
-  //       speed and output size.
-  //   1:  Consider two paths at each point when a backreference is found: using
-  //       the backreference or ignoring it.
-  //   2+: Consider further chains of paths at each point. Using values 2 or
-  //       greater for compression_level generally yields diminishing returns.
-  explicit PRSCompressor(ssize_t compression_level = 0, ProgressCallback progress_fn = nullptr);
-  ~PRSCompressor() = default;
-
-  // Adds more input data to be compressed, which logically comes after all
-  // previous data provided via add() calls. Cannot be called after close() is
-  // called.
-  void add(const void* data, size_t size);
-  void add(const std::string& data);
-
-  // Ends compression and returns the complete compressed result. It's OK to
-  // std::move() from the returned string reference.
-  std::string& close();
-
-  // Returns the total number of bytes passed to add() calls so far.
-  inline size_t input_size() const {
-    return this->input_bytes;
-  }
-
-private:
-  template <size_t Size>
-  struct WrappedLog {
-    parray<uint8_t, Size> data;
-
-    WrappedLog() : data(0) {}
-    ~WrappedLog() = default;
-
-    inline uint8_t at(size_t offset) const {
-      return this->data[offset % this->data.size()];
-    }
-    inline uint8_t& at(size_t offset) {
-      return this->data[offset % this->data.size()];
-    }
-  };
-
-  template <size_t Size>
-  struct IndexedLog : WrappedLog<Size> {
-    size_t offset;
-    size_t size;
-    std::array<std::deque<size_t>, 0x100> index;
-
-    IndexedLog()
-        : WrappedLog<Size>(),
-          offset(0),
-          size(0) {}
-    ~IndexedLog() = default;
-
-    inline size_t end_offset() const {
-      return this->offset + this->size;
-    }
-
-    void push_back(uint8_t v) {
-      if (this->size == Size) {
-        this->pop_front();
-      }
-      size_t write_offset = this->offset + this->size;
-      this->at(write_offset) = v;
-      this->index[v].push_back(write_offset);
-      this->size++;
-    }
-    uint8_t pop_back() {
-      if (!this->size) {
-        throw std::logic_error("pop_back called on empty IndexedLog");
-      }
-      this->size--;
-      size_t offset = this->offset + this->size;
-      uint8_t v = this->at(offset);
-      this->index[v].pop_back();
-      return v;
-    }
-    uint8_t pop_front() {
-      uint8_t v = this->at(this->offset);
-      this->index[v].pop_front();
-      this->offset++;
-      this->size--;
-      return v;
-    }
-    const std::deque<size_t>& find(uint8_t v) {
-      return this->index[v];
-    }
-  };
-
-  void add_byte(uint8_t v);
-  void advance();
-  void move_forward_data_to_reverse_log(size_t size);
-  void advance_literal();
-  void advance_short_copy(ssize_t offset, size_t size);
-  void advance_long_copy(ssize_t offset, size_t size);
-  void advance_extended_copy(ssize_t offset, size_t size);
-  void write_control(bool z);
-  void flush_control();
-
-  ssize_t compression_level;
-  ProgressCallback progress_fn;
-  bool closed;
-
-  size_t control_byte_offset;
-  uint16_t pending_control_bits;
-
-  size_t input_bytes;
-  WrappedLog<0x101> forward_log;
-  IndexedLog<0x2000> reverse_log;
-
-  StringWriter output;
-};
-
-// These functions use PRSCompressor to compress a buffer of data. This is
-// essentially a shortcut for constructing a PRSCompressor, calling .add() on
-// it once, then calling .close().
-std::string prs_compress(
-    const void* vdata,
-    size_t size,
-    ssize_t compression_level = 0,
-    ProgressCallback progress_fn = nullptr);
-std::string prs_compress(
-    const std::string& data,
-    ssize_t compression_level = 0,
-    ProgressCallback progress_fn = nullptr);
-
-// A faster form of prs_compress that doesn't have a tunable compression level.
-std::string prs_compress_indexed(
-    const void* vdata,
-    size_t size,
-    ProgressCallback progress_fn = nullptr);
-std::string prs_compress_indexed(
-    const std::string& data,
-    ProgressCallback progress_fn = nullptr);
-
-// Compresses data using PRS to the smallest possible output size. This function
-// is slow, but produces results significantly smaller than even Sega's original
-// compressor.
-std::string prs_compress_optimal(const void* vdata, size_t size, ProgressCallback progress_fn = nullptr);
-std::string prs_compress_optimal(const std::string& data, ProgressCallback progress_fn = nullptr);
-
-// Decompresses PRS-compressed data.
-struct PRSDecompressResult {
-  std::string data;
-  size_t input_bytes_used;
-};
-PRSDecompressResult prs_decompress_with_meta(const void* data, size_t size, size_t max_output_size = 0, bool allow_unterminated = false);
-PRSDecompressResult prs_decompress_with_meta(const std::string& data, size_t max_output_size = 0, bool allow_unterminated = false);
-std::string prs_decompress(const void* data, size_t size, size_t max_output_size = 0, bool allow_unterminated = false);
-std::string prs_decompress(const std::string& data, size_t max_output_size = 0, bool allow_unterminated = false);
-
-// Returns the decompressed size of PRS-compressed data, without actually
-// decompressing it.
-size_t prs_decompress_size(const void* data, size_t size, size_t max_output_size = 0, bool allow_unterminated = false);
-size_t prs_decompress_size(const std::string& data, size_t max_output_size = 0, bool allow_unterminated = false);
-
-// Prints the command stream from a PRS-compressed buffer.
-void prs_disassemble(FILE* stream, const void* data, size_t size);
-void prs_disassemble(FILE* stream, const std::string& data);
-
-////////////////////////////////////////////////////////////////////////////////
-// BC0 compression
-////////////////////////////////////////////////////////////////////////////////
-
-// Compresses data using the BC0 algorithm. Like with PRS, the optimal variant
-// is slow, but produces the smallest possible output.
-std::string bc0_compress_optimal(
-    const void* in_data_v,
-    size_t in_size,
-    ProgressCallback progress_fn = nullptr);
-std::string bc0_compress(const std::string& data, ProgressCallback progress_fn = nullptr);
-std::string bc0_compress(const void* in_data_v, size_t in_size, ProgressCallback progress_fn = nullptr);
-
-// Encodes data in a BC0-compatible format without compression (similar to using
-// compression_level=-1 with prs_compress).
-std::string bc0_encode(const void* in_data_v, size_t in_size);
-
-// Decompresses BC0-compressed data.
-std::string bc0_decompress(const std::string& data);
-std::string bc0_decompress(const void* data, size_t size);
-
-// Prints the command stream from a BC0-compressed buffer.
-void bc0_disassemble(FILE* stream, const std::string& data);
-void bc0_disassemble(FILE* stream, const void* data, size_t size);
+#pragma once
+
+#include <stddef.h>
+
+#include <array>
+#include <deque>
+#include <functional>
+#include <phosg/Tools.hh>
+#include <string>
+
+#include "Text.hh"
+
+enum class CompressPhase {
+  INDEX = 0,
+  CONSTRUCT_PATHS,
+  BACKTRACE_OPTIMAL_PATH,
+  GENERATE_RESULT,
+};
+
+template <>
+const char* phosg::name_for_enum<CompressPhase>(CompressPhase v);
+
+typedef std::function<void(CompressPhase phase, size_t input_progress, size_t input_size, size_t output_size)> ProgressCallback;
+
+////////////////////////////////////////////////////////////////////////////////
+// PRS compression
+////////////////////////////////////////////////////////////////////////////////
+
+// Use this class if you need to compress from multiple input buffers, or need
+// to compress multiple chunks and don't want to copy their contents
+// unnecessarily. (For most common use cases, use prs_compress, below, instead.)
+// To use this class, instantiate it, then call .add() one or more times, then
+// call .close() and use the returned string as the compressed result.
+class PRSCompressor {
+public:
+  // compression_level specifies how aggressively to search for alternate paths:
+  //   -1: Don't perform any compression at all, but produce output that can be
+  //       understood by prs_decompress. The output will be about 9/8 the size
+  //       of the input.
+  //   0:  Greedily search for the longest backreference at every point. Don't
+  //       consider any alternate paths. Generally offers a good balance between
+  //       speed and output size.
+  //   1:  Consider two paths at each point when a backreference is found: using
+  //       the backreference or ignoring it.
+  //   2+: Consider further chains of paths at each point. Using values 2 or
+  //       greater for compression_level generally yields diminishing returns.
+  explicit PRSCompressor(ssize_t compression_level = 0, ProgressCallback progress_fn = nullptr);
+  ~PRSCompressor() = default;
+
+  // Adds more input data to be compressed, which logically comes after all
+  // previous data provided via add() calls. Cannot be called after close() is
+  // called.
+  void add(const void* data, size_t size);
+  void add(const std::string& data);
+
+  // Ends compression and returns the complete compressed result. It's OK to
+  // std::move() from the returned string reference.
+  std::string& close();
+
+  // Returns the total number of bytes passed to add() calls so far.
+  inline size_t input_size() const {
+    return this->input_bytes;
+  }
+
+private:
+  template <size_t Size>
+  struct WrappedLog {
+    parray<uint8_t, Size> data;
+
+    WrappedLog() : data(0) {}
+    ~WrappedLog() = default;
+
+    inline uint8_t at(size_t offset) const {
+      return this->data[offset % this->data.size()];
+    }
+    inline uint8_t& at(size_t offset) {
+      return this->data[offset % this->data.size()];
+    }
+  };
+
+  template <size_t Size>
+  struct IndexedLog : WrappedLog<Size> {
+    size_t offset;
+    size_t size;
+    std::array<std::deque<size_t>, 0x100> index;
+
+    IndexedLog()
+        : WrappedLog<Size>(),
+          offset(0),
+          size(0) {}
+    ~IndexedLog() = default;
+
+    inline size_t end_offset() const {
+      return this->offset + this->size;
+    }
+
+    void push_back(uint8_t v) {
+      if (this->size == Size) {
+        this->pop_front();
+      }
+      size_t write_offset = this->offset + this->size;
+      this->at(write_offset) = v;
+      this->index[v].push_back(write_offset);
+      this->size++;
+    }
+    uint8_t pop_back() {
+      if (!this->size) {
+        throw std::logic_error("pop_back called on empty IndexedLog");
+      }
+      this->size--;
+      size_t offset = this->offset + this->size;
+      uint8_t v = this->at(offset);
+      this->index[v].pop_back();
+      return v;
+    }
+    uint8_t pop_front() {
+      uint8_t v = this->at(this->offset);
+      this->index[v].pop_front();
+      this->offset++;
+      this->size--;
+      return v;
+    }
+    const std::deque<size_t>& find(uint8_t v) {
+      return this->index[v];
+    }
+  };
+
+  void add_byte(uint8_t v);
+  void advance();
+  void move_forward_data_to_reverse_log(size_t size);
+  void advance_literal();
+  void advance_short_copy(ssize_t offset, size_t size);
+  void advance_long_copy(ssize_t offset, size_t size);
+  void advance_extended_copy(ssize_t offset, size_t size);
+  void write_control(bool z);
+  void flush_control();
+
+  ssize_t compression_level;
+  ProgressCallback progress_fn;
+  bool closed;
+
+  size_t control_byte_offset;
+  uint16_t pending_control_bits;
+
+  size_t input_bytes;
+  WrappedLog<0x101> forward_log;
+  IndexedLog<0x2000> reverse_log;
+
+  phosg::StringWriter output;
+};
+
+// These functions use PRSCompressor to compress a buffer of data. This is
+// essentially a shortcut for constructing a PRSCompressor, calling .add() on
+// it once, then calling .close().
+std::string prs_compress(
+    const void* vdata,
+    size_t size,
+    ssize_t compression_level = 0,
+    ProgressCallback progress_fn = nullptr);
+std::string prs_compress(
+    const std::string& data,
+    ssize_t compression_level = 0,
+    ProgressCallback progress_fn = nullptr);
+
+// A faster form of prs_compress that doesn't have a tunable compression level.
+std::string prs_compress_indexed(
+    const void* vdata,
+    size_t size,
+    ProgressCallback progress_fn = nullptr);
+std::string prs_compress_indexed(
+    const std::string& data,
+    ProgressCallback progress_fn = nullptr);
+
+// Compresses data using PRS to the smallest possible output size. This function
+// is slow, but produces results significantly smaller than even Sega's original
+// compressor.
+std::string prs_compress_optimal(const void* vdata, size_t size, ProgressCallback progress_fn = nullptr);
+std::string prs_compress_optimal(const std::string& data, ProgressCallback progress_fn = nullptr);
+
+// Compresses data using PRS to the LARGEST possible output size. There is no
+// practical use for this function except for amusement.
+std::string prs_compress_pessimal(const void* vdata, size_t size);
+
+// Decompresses PRS-compressed data.
+struct PRSDecompressResult {
+  std::string data;
+  size_t input_bytes_used;
+};
+PRSDecompressResult prs_decompress_with_meta(const void* data, size_t size, size_t max_output_size = 0, bool allow_unterminated = false);
+PRSDecompressResult prs_decompress_with_meta(const std::string& data, size_t max_output_size = 0, bool allow_unterminated = false);
+std::string prs_decompress(const void* data, size_t size, size_t max_output_size = 0, bool allow_unterminated = false);
+std::string prs_decompress(const std::string& data, size_t max_output_size = 0, bool allow_unterminated = false);
+
+// Returns the decompressed size of PRS-compressed data, without actually
+// decompressing it.
+size_t prs_decompress_size(const void* data, size_t size, size_t max_output_size = 0, bool allow_unterminated = false);
+size_t prs_decompress_size(const std::string& data, size_t max_output_size = 0, bool allow_unterminated = false);
+
+// Prints the command stream from a PRS-compressed buffer.
+void prs_disassemble(FILE* stream, const void* data, size_t size);
+void prs_disassemble(FILE* stream, const std::string& data);
+
+////////////////////////////////////////////////////////////////////////////////
+// BC0 compression
+////////////////////////////////////////////////////////////////////////////////
+
+// Compresses data using the BC0 algorithm. Like with PRS, the optimal variant
+// is slow, but produces the smallest possible output.
+std::string bc0_compress_optimal(
+    const void* in_data_v,
+    size_t in_size,
+    ProgressCallback progress_fn = nullptr);
+std::string bc0_compress(const std::string& data, ProgressCallback progress_fn = nullptr);
+std::string bc0_compress(const void* in_data_v, size_t in_size, ProgressCallback progress_fn = nullptr);
+
+// Encodes data in a BC0-compatible format without compression (similar to using
+// compression_level=-1 with prs_compress).
+std::string bc0_encode(const void* in_data_v, size_t in_size);
+
+// Decompresses BC0-compressed data.
+std::string bc0_decompress(const std::string& data);
+std::string bc0_decompress(const void* data, size_t size);
+
+// Prints the command stream from a BC0-compressed buffer.
+void bc0_disassemble(FILE* stream, const std::string& data);
+void bc0_disassemble(FILE* stream, const void* data, size_t size);

src/DCSerialNumbers.cc

@@ -2,6 +2,7 @@
 
 #include <stdint.h>
 
+#include <mutex>
 #include <phosg/Random.hh>
 #include <phosg/Strings.hh>
 #include <phosg/Time.hh>
@@ -24,6 +25,7 @@ static const uint32_t primes1[] = {
     0x313, 0x31D, 0x329, 0x32B, 0x335, 0x337, 0x33B, 0x33D, 0x347, 0x355, 0x359,
     0x35B, 0x35F, 0x36D, 0x371, 0x373, 0x377, 0x38B, 0x38F, 0x397, 0x3A1, 0x3A9,
     0x3AD, 0x3B3, 0x3B9, 0x3C7, 0x3CB, 0x3D1, 0x3D7, 0x3DF, 0x3E5};
+
 static const uint32_t primes2[] = {
     0x3F1, 0x3F5, 0x3FB, 0x3FD, 0x407, 0x409, 0x40F, 0x419, 0x41B, 0x425, 0x427,
     0x42D, 0x43F, 0x443, 0x445, 0x449, 0x44F, 0x455, 0x45D, 0x463, 0x469, 0x47F,
@@ -135,6 +137,8 @@ static const uint32_t primes2[] = {
     0x2627, 0x2629, 0x2635, 0x263B, 0x263F, 0x264B, 0x2653, 0x2659, 0x2665,
     0x2669, 0x266F, 0x267B, 0x2681, 0x2683, 0x268F, 0x269B, 0x269F, 0x26AD,
     0x26B3, 0x26C3, 0x26C9, 0x26CB, 0x26D5, 0x26DD, 0x26EF, 0x26F5};
+static constexpr size_t num_primes2 = sizeof(primes2) / sizeof(primes2[0]);
+
 static const uint32_t primes3[] = {
     0x2717, 0x2719, 0x2735, 0x2737, 0x274D, 0x2753, 0x2755, 0x275F, 0x276B,
     0x276D, 0x2773, 0x2777, 0x277F, 0x2795, 0x279B, 0x279D, 0x27A7, 0x27AF,
@@ -1107,15 +1111,19 @@ static const uint32_t primes3[] = {
     0x18569, 0x1857B, 0x1857D, 0x18581, 0x18587, 0x18589, 0x18595, 0x185B1,
     0x185B7, 0x185CB, 0x185D1, 0x185E1, 0x185E9, 0x185EF, 0x185F5, 0x185F9,
     0x185FF, 0x18613, 0x1861F};
+static constexpr size_t num_primes3 = sizeof(primes3) / sizeof(primes3[0]);
 
 static bool check_prime3(uint64_t prime3) {
   static vector<bool> primes3_set;
+  static mutex primes3_init_mutex;
   if (primes3_set.empty()) {
-    size_t num_primes3 = sizeof(primes3) / sizeof(primes3[0]);
-    size_t primes3_set_size = primes3[num_primes3 - 1] - primes3[0] + 1;
-    primes3_set.resize(primes3_set_size, false);
-    for (size_t z = 0; z < num_primes3; z++) {
-      primes3_set[primes3[z] - primes3[0]] = true;
+    lock_guard g(primes3_init_mutex);
+    if (primes3_set.empty()) {
+      size_t primes3_set_size = primes3[num_primes3 - 1] - primes3[0] + 1;
+      primes3_set.resize(primes3_set_size, false);
+      for (size_t z = 0; z < num_primes3; z++) {
+        primes3_set[primes3[z] - primes3[0]] = true;
+      }
     }
   }
   uint64_t offset = prime3 - primes3[0];
@@ -1174,7 +1182,7 @@ static uint64_t decode_dc_serial_number_str(const string& s) {
     if (new_ch == '\0') {
       return INVALID_PRODUCT;
     }
-    serial_number = (serial_number << 4) | value_for_hex_char(new_ch);
+    serial_number = (serial_number << 4) | phosg::value_for_hex_char(new_ch);
   }
   return serial_number;
 }
@@ -1227,8 +1235,8 @@ bool dc_serial_number_is_valid_slow(const string& s, uint8_t domain, uint8_t sub
   }
 
   for (; offset1 < limit1; offset1++) {
-    for (size_t offset2 = 0; offset2 < sizeof(primes2) / sizeof(primes2[0]); offset2++) {
-      for (size_t offset3 = 0; offset3 < sizeof(primes3) / sizeof(primes3[0]); offset3++) {
+    for (size_t offset2 = 0; offset2 < num_primes2; offset2++) {
+      for (size_t offset3 = 0; offset3 < num_primes3; offset3++) {
         if (primes1[offset1] * primes2[offset2] * primes3[offset3] == serial_number) {
           return true;
         }
@@ -1251,7 +1259,7 @@ bool decoded_dc_serial_number_is_valid_fast(uint32_t serial_number, uint8_t doma
         continue;
       }
       uint64_t sub1 = sub0 / primes1[offset1];
-      for (size_t offset2 = 0; offset2 < sizeof(primes2) / sizeof(primes2[0]); offset2++) {
+      for (size_t offset2 = 0; offset2 < num_primes2; offset2++) {
         if (sub1 % primes2[offset2]) {
           continue;
         }
@@ -1291,12 +1299,12 @@ string generate_dc_serial_number(uint8_t domain, uint8_t subdomain) {
     throw runtime_error("invalid domain");
   }
 
-  size_t det1 = (subdomain == 0xFF) ? random_object<uint32_t>() : subdomain;
+  size_t det1 = (subdomain == 0xFF) ? phosg::random_object<uint32_t>() : subdomain;
   size_t index1 = offset1 + (det1 % (limit1 - offset1));
-  size_t index2 = random_object<uint32_t>() % (sizeof(primes2) / sizeof(primes2[0]));
-  size_t index3 = random_object<uint32_t>() % (sizeof(primes3) / sizeof(primes3[0]));
+  size_t index2 = phosg::random_object<uint32_t>() % num_primes2;
+  size_t index3 = phosg::random_object<uint32_t>() % num_primes3;
   uint32_t value = primes1[index1] * primes2[index2] * primes3[index3];
-  string s = string_printf("%08X", value);
+  string s = phosg::string_printf("%08X", value);
 
   string ret;
   for (char ch : s) {
@@ -1306,56 +1314,81 @@ string generate_dc_serial_number(uint8_t domain, uint8_t subdomain) {
 }
 
 unordered_map<uint32_t, string> generate_all_dc_serial_numbers(uint8_t domain, uint8_t subdomain) {
-  vector<uint8_t> domains;
-  if (domain == 0xFF) {
-    domains.emplace_back(0x00);
-    domains.emplace_back(0x01);
-    domains.emplace_back(0x02);
-  } else {
-    domains.emplace_back(domain);
+  DCSerialNumberIterator iter;
+
+  if (domain < 3) {
+    iter.domain = domain;
+    iter.end_domain = domain + 1;
+  } else if (domain != 0xFF) {
+    throw runtime_error("invalid domain");
   }
 
-  vector<uint8_t> subdomains;
-  if (subdomain == 0xFF) {
-    subdomains.emplace_back(0x00);
-    subdomains.emplace_back(0x01);
-    subdomains.emplace_back(0x02);
-  } else {
-    subdomains.emplace_back(subdomain);
+  if (subdomain < 3) {
+    iter.subdomain = subdomain;
+    iter.start_subdomain = subdomain;
+    iter.end_subdomain = subdomain + 1;
+  } else if (subdomain != 0xFF) {
+    throw runtime_error("invalid subdomain");
   }
 
+  uint32_t serial_number;
   unordered_map<uint32_t, string> ret;
-  for (uint8_t domain : domains) {
-    size_t offset1, limit1;
-    if (domain == 0) {
-      offset1 = 0x00;
-      limit1 = 0x03;
-    } else if (domain == 1) {
-      offset1 = 0x1E;
-      limit1 = 0x21;
-    } else if (domain == 2) {
-      offset1 = 0x3C;
-      limit1 = 0x3F;
-    } else {
-      throw runtime_error("invalid domain");
-    }
-
-    for (uint8_t subdomain : subdomains) {
-      size_t index1 = offset1 + (subdomain % (limit1 - offset1));
-      for (size_t index2 = 0; index2 < sizeof(primes2) / sizeof(primes2[0]); index2++) {
-        for (size_t index3 = 0; index3 < sizeof(primes3) / sizeof(primes3[0]); index3++) {
-          uint32_t value = primes1[index1] * primes2[index2] * primes3[index3];
-          ret[encode_dc_serial_number_int(value)].push_back(((domain << 2) & 3) | (subdomain & 3));
-        }
-        fprintf(stderr, "... domain=%hhu subdomain=%hhu index2=%zu results=%zu (0x%zX)\n", domain, subdomain, index2, ret.size(), ret.size());
-      }
+  while ((serial_number = iter.next()) != 0) {
+    ret[serial_number].push_back(((iter.domain << 2) & 3) | (iter.subdomain & 3));
+    if (iter.index3 == 0) {
+      fprintf(stderr, "... (it) domain=%hhu subdomain=%hhu index2=%hu results=%zu (0x%zX)\n", iter.domain, iter.subdomain, iter.index2, ret.size(), ret.size());
     }
   }
   return ret;
 }
 
+uint32_t DCSerialNumberIterator::next() {
+  if (!this->started) {
+    this->started = true;
+  } else if (!this->complete) {
+    this->index3++;
+    if (this->index3 >= num_primes3) {
+      this->index3 = 0;
+      this->index2++;
+      if (this->index2 >= num_primes2) {
+        this->index2 = 0;
+        this->subdomain++;
+        if (this->subdomain >= this->end_subdomain) {
+          this->subdomain = this->start_subdomain;
+          this->domain++;
+          if (this->domain >= this->end_domain) {
+            this->serial_number = 0;
+            this->complete = true;
+          }
+        }
+      }
+    }
+  }
+  if (!this->complete) {
+    size_t index1 = (30 * this->domain) + (this->subdomain % 3);
+    return encode_dc_serial_number_int(primes1[index1] * primes2[this->index2] * primes3[this->index3]);
+  } else {
+    return 0;
+  }
+}
+
+size_t DCSerialNumberIterator::total_count() const {
+  return (this->end_domain - this->start_domain) * (this->end_subdomain - this->start_subdomain) * num_primes2 * num_primes3;
+}
+
+size_t DCSerialNumberIterator::progress() const {
+  size_t domains_done = this->domain - this->start_domain;
+  size_t subdomains_per_domain = this->end_subdomain - this->start_subdomain;
+  size_t subdomains_done = this->subdomain - this->start_subdomain;
+  return (
+      (domains_done * subdomains_per_domain * num_primes2 * num_primes3) +
+      (subdomains_done * num_primes2 * num_primes3) +
+      (this->index2 * num_primes3) +
+      this->index3);
+}
+
 void dc_serial_number_speed_test(uint64_t seed) {
-  uint32_t effective_seed = (seed & 0xFFFFFFFF00000000) ? random_object<uint32_t>() : seed;
+  uint32_t effective_seed = (seed & 0xFFFFFFFF00000000) ? phosg::random_object<uint32_t>() : seed;
   fprintf(stderr, "Product speed test with seed=%08" PRIX32 "\n", effective_seed);
   PSOV2Encryption crypt(effective_seed);
   uint64_t time_slow = 0;
@@ -1363,15 +1396,15 @@ void dc_serial_number_speed_test(uint64_t seed) {
   size_t num_disagreements = 0;
   static constexpr size_t count = 0x1000;
   for (size_t z = 0; z < count; z++) {
-    string s = string_printf("%08X", crypt.next());
+    string s = phosg::string_printf("%08X", crypt.next());
 
-    uint64_t start = now();
+    uint64_t start = phosg::now();
     bool is_valid_fast = dc_serial_number_is_valid_fast(s, 1, 0xFF);
-    time_fast += now() - start;
+    time_fast += phosg::now() - start;
 
-    start = now();
+    start = phosg::now();
     bool is_valid_slow = dc_serial_number_is_valid_slow(s, 1, 0xFF);
-    time_slow += now() - start;
+    time_slow += phosg::now() - start;
 
     if (((z & 0xF) == 0) || is_valid_slow || is_valid_fast) {
       fprintf(stderr, "... %02zX: %s => %s %s%s\n", z, s.c_str(), is_valid_slow ? "SLOW" : "----", is_valid_fast ? "FAST" : "----", is_valid_slow != is_valid_fast ? " !!!" : "");
@@ -1386,3 +1419,74 @@ void dc_serial_number_speed_test(uint64_t seed) {
   fprintf(stderr, "Fast vs. slow speedup: %zux\n", static_cast<size_t>(time_slow / time_fast));
   fprintf(stderr, "Disagreements: %zu\n", num_disagreements);
 }
+
+string decrypt_dp_address_jpn(
+    const string& executable,
+    const string& values,
+    const string& indexes) {
+  phosg::StringReader values_r(values);
+  phosg::StringReader indexes_r(indexes);
+
+  size_t fixup_values_offset = values_r.pget_u32l(0x3FFC) - 0x8C004000;
+  size_t fixup_steps_offset = indexes_r.pget_u32l(0x3BFC) - 0x8C008400;
+  phosg::StringReader fixup_values_r = values_r.sub(fixup_values_offset);
+  phosg::StringReader fixup_steps_r = indexes_r.sub(fixup_steps_offset);
+
+  auto decrypted = decrypt_pr2_data<false>(executable);
+  size_t fixup_offset = 0;
+  while (fixup_steps_r.get_u8(false)) {
+    fixup_offset += (fixup_steps_r.get_u8() << 2);
+    fixup_steps_r.skip(1);
+    if (fixup_offset + 4 > decrypted.compressed_data.size()) {
+      throw runtime_error("fixup beyond end of compressed data");
+    }
+    *reinterpret_cast<le_uint32_t*>(decrypted.compressed_data.data() + fixup_offset) = fixup_values_r.get_u32l();
+  }
+
+  return prs_decompress(decrypted.compressed_data);
+}
+
+EncryptedDCv2Executables encrypt_dp_address_jpn(const string& executable, const string& indexes) {
+  EncryptedDCv2Executables ret;
+
+  string compressed = prs_compress(executable);
+  ret.executable = encrypt_pr2_data<false>(compressed, executable.size(), phosg::random_object<uint32_t>() & 0x7FFFFF7F);
+
+  phosg::StringReader indexes_r(indexes);
+  size_t fixup_steps_offset = indexes_r.pget_u32l(0x3BFC) - 0x8C008400;
+  ret.indexes = indexes;
+  ret.indexes.at(fixup_steps_offset) = 0;
+  return ret;
+}
+
+std::string crypt_dp_address_jpn_simple(const std::string& data, int64_t mask_key) {
+  if (data.size() & 3) {
+    throw runtime_error("size is not a multiple of 4");
+  }
+
+  phosg::StringReader r(data);
+  if (mask_key < 0) {
+    unordered_map<uint32_t, size_t> key_freq;
+    while (!r.eof()) {
+      key_freq[r.get_u32l()] += 1;
+    }
+    size_t max_v = 0;
+    for (const auto& it : key_freq) {
+      if (it.second > max_v) {
+        max_v = it.second;
+        mask_key = it.first;
+      }
+    }
+    if (mask_key < 0) {
+      throw runtime_error("cannot determine mask key");
+    }
+    phosg::log_info("Determined %08" PRIX64 " to be the most likely mask key", mask_key);
+    r.go(0);
+  }
+
+  phosg::StringWriter w;
+  while (!r.eof()) {
+    w.put_u32l(r.get_u32l() ^ mask_key);
+  }
+  return std::move(w.str());
+}

src/DCSerialNumbers.hh

@@ -20,4 +20,36 @@ bool decoded_dc_serial_number_is_valid_fast(
 std::string generate_dc_serial_number(uint8_t domain, uint8_t subdomain = 0xFF);
 std::unordered_map<uint32_t, std::string> generate_all_dc_serial_numbers(uint8_t domain = 0xFF, uint8_t subdomain = 0xFF);
 
+struct DCSerialNumberIterator {
+  bool started = false;
+  bool complete = false;
+  uint8_t domain = 0;
+  uint8_t start_domain = 0;
+  uint8_t end_domain = 3;
+  uint8_t subdomain = 0;
+  uint8_t start_subdomain = 0;
+  uint8_t end_subdomain = 3;
+  uint16_t index2 = 0;
+  uint16_t index3 = 0;
+  uint32_t serial_number = 0;
+
+  uint32_t next();
+
+  size_t total_count() const;
+  size_t progress() const;
+};
+
 void dc_serial_number_speed_test(uint64_t seed = 0xFFFFFFFFFFFFFFFF);
+
+struct EncryptedDCv2Executables {
+  std::string executable;
+  std::string indexes;
+};
+
+std::string decrypt_dp_address_jpn(
+    const std::string& dp_address_jpn_data,
+    const std::string& iwashi_sea_data,
+    const std::string& katsuo_sea_data);
+EncryptedDCv2Executables encrypt_dp_address_jpn(const std::string& executable, const std::string& indexes);
+
+std::string crypt_dp_address_jpn_simple(const std::string& data, int64_t seed = -1);

src/DNSServer.cc

@@ -1,118 +1,121 @@
-#include "DNSServer.hh"
-
-#include <netinet/in.h>
-#include <poll.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <phosg/Encoding.hh>
-#include <phosg/Network.hh>
-#include <phosg/Strings.hh>
-#include <string>
-#include <vector>
-
-#include "Loggers.hh"
-#include "NetworkAddresses.hh"
-
-using namespace std;
-
-DNSServer::DNSServer(
-    shared_ptr<struct event_base> base,
-    uint32_t local_connect_address, uint32_t external_connect_address)
-    : base(base),
-      local_connect_address(local_connect_address),
-      external_connect_address(external_connect_address) {}
-
-DNSServer::~DNSServer() {
-  for (const auto& it : this->fd_to_receive_event) {
-    close(it.first);
-  }
-}
-
-void DNSServer::listen(const std::string& socket_path) {
-  this->add_socket(::listen(socket_path, 0, 0));
-}
-
-void DNSServer::listen(const std::string& addr, int port) {
-  this->add_socket(::listen(addr, port, 0));
-}
-
-void DNSServer::listen(int port) {
-  this->add_socket(::listen("", port, 0));
-}
-
-void DNSServer::add_socket(int fd) {
-  unique_ptr<struct event, void (*)(struct event*)> e(
-      event_new(this->base.get(), fd, EV_READ | EV_PERSIST, &DNSServer::dispatch_on_receive_message, this),
-      event_free);
-  event_add(e.get(), nullptr);
-  this->fd_to_receive_event.emplace(fd, std::move(e));
-}
-
-void DNSServer::dispatch_on_receive_message(evutil_socket_t fd,
-    short events, void* ctx) {
-  reinterpret_cast<DNSServer*>(ctx)->on_receive_message(fd, events);
-}
-
-string DNSServer::response_for_query(
-    const void* vdata, size_t size, uint32_t resolved_address) {
-  if (size < 0x0C) {
-    throw invalid_argument("query too small");
-  }
-
-  const char* data = reinterpret_cast<const char*>(vdata);
-  size_t name_len = strlen(&data[12]) + 1;
-
-  be_uint32_t be_resolved_address = resolved_address;
-
-  string response;
-  response.append(data, 2);
-  response.append("\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00", 10);
-  response.append(&data[12], name_len);
-  response.append("\x00\x01\x00\x01\xC0\x0C\x00\x01\x00\x01\x00\x00\x00\x3C\x00\x04", 16);
-  response.append(reinterpret_cast<const char*>(&be_resolved_address), 4);
-  return response;
-}
-
-string DNSServer::response_for_query(
-    const string& query, uint32_t resolved_address) {
-  return DNSServer::response_for_query(query.data(), query.size(), resolved_address);
-}
-
-void DNSServer::on_receive_message(int fd, short) {
-  for (;;) {
-    sockaddr_in remote;
-    socklen_t remote_size = sizeof(sockaddr_in);
-    memset(&remote, 0, remote_size);
-
-    string input(2048, 0);
-    ssize_t bytes = recvfrom(fd, const_cast<char*>(input.data()), input.size(),
-        0, reinterpret_cast<sockaddr*>(&remote), &remote_size);
-
-    if (bytes < 0) {
-      if (errno != EAGAIN) {
-        dns_server_log.error("input error %d", errno);
-        throw runtime_error("cannot read from udp socket");
-      }
-      break;
-
-    } else if (bytes == 0) {
-      break;
-
-    } else if (bytes < 0x0C) {
-      dns_server_log.warning("input query too small");
-      print_data(stderr, input.data(), bytes);
-
-    } else {
-      input.resize(bytes);
-      uint32_t remote_address = ntohl(remote.sin_addr.s_addr);
-      uint32_t connect_address = is_local_address(remote_address)
-          ? this->local_connect_address
-          : this->external_connect_address;
-      string response = this->response_for_query(input, connect_address);
-      sendto(fd, response.data(), response.size(), 0,
-          reinterpret_cast<const sockaddr*>(&remote), remote_size);
-    }
-  }
-}
+#include "DNSServer.hh"
+
+#include <netinet/in.h>
+#include <poll.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <phosg/Encoding.hh>
+#include <phosg/Network.hh>
+#include <phosg/Strings.hh>
+#include <string>
+#include <vector>
+
+#include "Loggers.hh"
+#include "NetworkAddresses.hh"
+
+using namespace std;
+
+DNSServer::DNSServer(
+    shared_ptr<struct event_base> base,
+    uint32_t local_connect_address,
+    uint32_t external_connect_address,
+    shared_ptr<const IPV4RangeSet> banned_ipv4_ranges)
+    : base(base),
+      local_connect_address(local_connect_address),
+      external_connect_address(external_connect_address),
+      banned_ipv4_ranges(banned_ipv4_ranges) {}
+
+DNSServer::~DNSServer() {
+  for (const auto& it : this->fd_to_receive_event) {
+    close(it.first);
+  }
+}
+
+void DNSServer::listen(const std::string& socket_path) {
+  this->add_socket(phosg::listen(socket_path, 0, 0));
+}
+
+void DNSServer::listen(const std::string& addr, int port) {
+  this->add_socket(phosg::listen(addr, port, 0));
+}
+
+void DNSServer::listen(int port) {
+  this->add_socket(phosg::listen("", port, 0));
+}
+
+void DNSServer::add_socket(int fd) {
+  unique_ptr<struct event, void (*)(struct event*)> e(
+      event_new(this->base.get(), fd, EV_READ | EV_PERSIST, &DNSServer::dispatch_on_receive_message, this),
+      event_free);
+  event_add(e.get(), nullptr);
+  this->fd_to_receive_event.emplace(fd, std::move(e));
+}
+
+void DNSServer::dispatch_on_receive_message(evutil_socket_t fd,
+    short events, void* ctx) {
+  reinterpret_cast<DNSServer*>(ctx)->on_receive_message(fd, events);
+}
+
+string DNSServer::response_for_query(const void* vdata, size_t size, uint32_t resolved_address) {
+  if (size < 0x0C) {
+    throw invalid_argument("query too small");
+  }
+
+  const char* data = reinterpret_cast<const char*>(vdata);
+  size_t name_len = strlen(&data[12]) + 1;
+
+  phosg::be_uint32_t be_resolved_address = resolved_address;
+
+  string response;
+  response.append(data, 2);
+  response.append("\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00", 10);
+  response.append(&data[12], name_len);
+  response.append("\x00\x01\x00\x01\xC0\x0C\x00\x01\x00\x01\x00\x00\x00\x3C\x00\x04", 16);
+  response.append(reinterpret_cast<const char*>(&be_resolved_address), 4);
+  return response;
+}
+
+string DNSServer::response_for_query(
+    const string& query, uint32_t resolved_address) {
+  return DNSServer::response_for_query(query.data(), query.size(), resolved_address);
+}
+
+void DNSServer::on_receive_message(int fd, short) {
+  for (;;) {
+    struct sockaddr_storage remote;
+    socklen_t remote_size = sizeof(sockaddr_in);
+    memset(&remote, 0, remote_size);
+
+    string input(2048, 0);
+    ssize_t bytes = recvfrom(fd, const_cast<char*>(input.data()), input.size(),
+        0, reinterpret_cast<sockaddr*>(&remote), &remote_size);
+
+    if (bytes < 0) {
+      if (errno != EAGAIN) {
+        dns_server_log.error("input error %d", errno);
+        throw runtime_error("cannot read from udp socket");
+      }
+      break;
+
+    } else if (bytes == 0) {
+      break;
+
+    } else if (bytes < 0x0C) {
+      dns_server_log.warning("input query too small");
+      phosg::print_data(stderr, input.data(), bytes);
+
+    } else if (!this->banned_ipv4_ranges->check(remote)) {
+      input.resize(bytes);
+      const sockaddr_in* remote_sin = reinterpret_cast<const sockaddr_in*>(&remote);
+      uint32_t remote_address = ntohl(remote_sin->sin_addr.s_addr);
+      uint32_t connect_address = is_local_address(remote_address)
+          ? this->local_connect_address
+          : this->external_connect_address;
+      string response = this->response_for_query(input, connect_address);
+      sendto(fd, response.data(), response.size(), 0,
+          reinterpret_cast<const sockaddr*>(&remote), remote_size);
+    }
+  }
+}

src/DNSServer.hh

@@ -1,36 +1,44 @@
-#pragma once
-
-#include <event2/event.h>
-
-#include <memory>
-#include <set>
-#include <string>
-#include <unordered_map>
-
-class DNSServer {
-public:
-  DNSServer(std::shared_ptr<struct event_base> base,
-      uint32_t local_connect_address, uint32_t external_connect_address);
-  DNSServer(const DNSServer&) = delete;
-  DNSServer(DNSServer&&) = delete;
-  virtual ~DNSServer();
-
-  void listen(const std::string& socket_path);
-  void listen(const std::string& addr, int port);
-  void listen(int port);
-  void add_socket(int fd);
-
-  static std::string response_for_query(
-      const void* vdata, size_t size, uint32_t resolved_address);
-  static std::string response_for_query(
-      const std::string& query, uint32_t resolved_address);
-
-private:
-  std::shared_ptr<struct event_base> base;
-  std::unordered_map<int, std::unique_ptr<struct event, void (*)(struct event*)>> fd_to_receive_event;
-  uint32_t local_connect_address;
-  uint32_t external_connect_address;
-
-  static void dispatch_on_receive_message(evutil_socket_t fd, short events, void* ctx);
-  void on_receive_message(int fd, short event);
-};
+#pragma once
+
+#include <event2/event.h>
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+
+#include "IPV4RangeSet.hh"
+
+class DNSServer {
+public:
+  DNSServer(
+      std::shared_ptr<struct event_base> base,
+      uint32_t local_connect_address,
+      uint32_t external_connect_address,
+      std::shared_ptr<const IPV4RangeSet> banned_ipv4_ranges);
+  DNSServer(const DNSServer&) = delete;
+  DNSServer(DNSServer&&) = delete;
+  virtual ~DNSServer();
+
+  inline void set_banned_ipv4_ranges(std::shared_ptr<const IPV4RangeSet> banned_ipv4_ranges) {
+    this->banned_ipv4_ranges = banned_ipv4_ranges;
+  }
+
+  void listen(const std::string& socket_path);
+  void listen(const std::string& addr, int port);
+  void listen(int port);
+  void add_socket(int fd);
+
+  static std::string response_for_query(const void* vdata, size_t size, uint32_t resolved_address);
+  static std::string response_for_query(const std::string& query, uint32_t resolved_address);
+
+private:
+  std::shared_ptr<struct event_base> base;
+  std::unordered_map<int, std::unique_ptr<struct event, void (*)(struct event*)>> fd_to_receive_event;
+  uint32_t local_connect_address;
+  uint32_t external_connect_address;
+  std::shared_ptr<const IPV4RangeSet> banned_ipv4_ranges;
+
+  static void dispatch_on_receive_message(evutil_socket_t fd, short events, void* ctx);
+  void on_receive_message(int fd, short event);
+};

src/DownloadSession.cc

@@ -0,0 +1,870 @@
+#include "DownloadSession.hh"
+
+#include <arpa/inet.h>
+#include <ctype.h>
+#include <errno.h>
+#include <event2/buffer.h>
+#include <event2/bufferevent.h>
+#include <event2/event.h>
+#include <event2/listener.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <iostream>
+#include <phosg/Encoding.hh>
+#include <phosg/Filesystem.hh>
+#include <phosg/Network.hh>
+#include <phosg/Random.hh>
+#include <phosg/Strings.hh>
+#include <phosg/Time.hh>
+
+#include "Loggers.hh"
+#include "PSOProtocol.hh"
+#include "ProxyCommands.hh"
+#include "ReceiveCommands.hh"
+#include "ReceiveSubcommands.hh"
+#include "SendCommands.hh"
+
+using namespace std;
+
+static string random_name() {
+  string ret;
+  size_t length = (phosg::random_object<size_t>() % 12) + 4;
+  static const string alphabet = "QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm1234567890-+<>:\"\',.";
+  while (ret.size() < length) {
+    ret.push_back(alphabet[phosg::random_object<size_t>() % alphabet.size()]);
+  }
+  return ret;
+}
+
+DownloadSession::DownloadSession(
+    std::shared_ptr<struct event_base> base,
+    const struct sockaddr_storage& remote,
+    const std::string& output_dir,
+    Version version,
+    uint8_t language,
+    std::shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file,
+    uint32_t hardware_id,
+    uint32_t serial_number,
+    const std::string& access_key,
+    const std::string& username,
+    const std::string& password,
+    const std::string& xb_gamertag,
+    uint64_t xb_user_id,
+    uint64_t xb_account_id,
+    std::shared_ptr<PSOBBCharacterFile> character,
+    const std::unordered_set<std::string>& ship_menu_selections,
+    const std::vector<std::string>& on_request_complete_commands,
+    bool interactive,
+    bool show_command_data)
+    : output_dir(output_dir),
+      bb_key_file(bb_key_file),
+      hardware_id(hardware_id),
+      serial_number(serial_number),
+      access_key(access_key),
+      username(username),
+      password(password),
+      xb_gamertag(xb_gamertag),
+      xb_user_id(xb_user_id),
+      xb_account_id(xb_account_id),
+      character(character),
+      ship_menu_selections(ship_menu_selections),
+      on_request_complete_commands(on_request_complete_commands),
+      interactive(interactive),
+      log(phosg::string_printf("[DownloadSession:%s] ", phosg::name_for_enum(version)), proxy_server_log.min_level),
+      base(base),
+      channel(
+          version,
+          language,
+          DownloadSession::dispatch_on_channel_input,
+          DownloadSession::dispatch_on_channel_error,
+          this,
+          phosg::render_sockaddr_storage(remote),
+          show_command_data ? phosg::TerminalFormat::FG_GREEN : phosg::TerminalFormat::END,
+          show_command_data ? phosg::TerminalFormat::FG_YELLOW : phosg::TerminalFormat::END),
+      guild_card_number(0),
+      prev_cmd_data(0),
+      client_config(0),
+      sent_96(false),
+      should_request_category_list(true),
+      current_request(0),
+      current_game_config_index(0),
+      in_game(false),
+      bin_complete(false),
+      dat_complete(false) {
+  if (this->output_dir.empty()) {
+    this->output_dir = ".";
+  }
+
+  switch (this->channel.version) {
+    case Version::DC_V1:
+    case Version::DC_V2:
+      if (this->hardware_id == 0 || this->serial_number == 0 || this->access_key.empty()) {
+        throw runtime_error("missing credentials");
+      }
+      break;
+    case Version::PC_V2:
+      if (this->serial_number == 0 || this->access_key.empty()) {
+        throw runtime_error("missing credentials");
+      }
+      break;
+    case Version::GC_V3:
+      if (this->serial_number == 0 || this->access_key.empty() || this->password.empty()) {
+        throw runtime_error("missing credentials");
+      }
+      break;
+    case Version::XB_V3:
+      if (this->xb_gamertag.empty() || this->xb_user_id == 0 || this->xb_account_id == 0) {
+        throw runtime_error("missing credentials");
+      }
+      break;
+    case Version::BB_V4:
+      if (this->username.empty() || this->password.empty()) {
+        throw runtime_error("missing credentials");
+      }
+      break;
+    default:
+      throw runtime_error("unsupported version");
+  }
+
+  this->character->inventory.language = this->channel.language;
+
+  if (remote.ss_family != AF_INET) {
+    throw runtime_error("remote is not AF_INET");
+  }
+
+  string netloc_str = phosg::render_sockaddr_storage(remote);
+  this->log.info("Connecting to %s", netloc_str.c_str());
+
+  struct bufferevent* bev = bufferevent_socket_new(
+      this->base.get(), -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS);
+  if (!bev) {
+    throw runtime_error(phosg::string_printf("failed to open socket (%d)", EVUTIL_SOCKET_ERROR()));
+  }
+  this->channel.set_bufferevent(bev, 0);
+
+  if (bufferevent_socket_connect(this->channel.bev.get(),
+          reinterpret_cast<const sockaddr*>(&remote), sizeof(struct sockaddr_in)) != 0) {
+    throw runtime_error(phosg::string_printf("failed to connect (%d)", EVUTIL_SOCKET_ERROR()));
+  }
+}
+
+void DownloadSession::dispatch_on_channel_input(Channel& ch, uint16_t command, uint32_t flag, std::string& data) {
+  auto* session = reinterpret_cast<DownloadSession*>(ch.context_obj);
+  session->on_channel_input(command, flag, data);
+}
+
+void DownloadSession::send_93_9D_9E(bool extended) {
+  if (is_v1(this->channel.version)) {
+    C_LoginExtendedV1_DC_93 ret;
+    ret.player_tag = this->guild_card_number ? 0xFFFF0000 : 0x00010000;
+    ret.guild_card_number = this->guild_card_number;
+    ret.sub_version = default_sub_version_for_version(this->channel.version);
+    ret.is_extended = extended ? 1 : 0;
+    ret.language = this->channel.language;
+    ret.serial_number.encode(phosg::string_printf("%08" PRIX32, this->serial_number));
+    ret.access_key.encode(this->access_key);
+    ret.hardware_id.encode(phosg::string_printf("%08" PRIX32, this->hardware_id));
+    ret.name.encode(this->character->disp.name.decode());
+    this->channel.send(0x93, 0x01, &ret, extended ? sizeof(ret) : sizeof(C_LoginV1_DC_93));
+
+  } else if (is_v2(this->channel.version)) {
+    C_LoginExtended_PC_9D ret;
+    ret.player_tag = this->guild_card_number ? 0xFFFF0000 : 0x00010000;
+    ret.guild_card_number = this->guild_card_number;
+    ret.sub_version = default_sub_version_for_version(this->channel.version);
+    ret.is_extended = extended ? 1 : 0;
+    ret.language = this->channel.language;
+    ret.serial_number.encode(phosg::string_printf("%08" PRIX32, this->serial_number));
+    ret.access_key.encode(this->access_key);
+    ret.serial_number2 = ret.serial_number;
+    ret.access_key2 = ret.access_key;
+    ret.name.encode(this->character->disp.name.decode());
+    size_t data_size = extended
+        ? ((this->channel.version == Version::PC_V2) ? sizeof(ret) : sizeof(C_LoginExtended_DC_GC_9D))
+        : sizeof(C_Login_DC_PC_GC_9D);
+    this->channel.send(0x9D, 0x01, &ret, data_size);
+
+  } else if (this->channel.version == Version::GC_V3) {
+    C_LoginExtended_GC_9E ret;
+    ret.player_tag = this->guild_card_number ? 0xFFFF0000 : 0x00010000;
+    ret.guild_card_number = this->guild_card_number;
+    ret.sub_version = default_sub_version_for_version(this->channel.version);
+    ret.is_extended = extended ? 1 : 0;
+    ret.language = this->channel.language;
+    ret.serial_number.encode(phosg::string_printf("%08" PRIX32, this->serial_number));
+    ret.access_key.encode(this->access_key);
+    ret.serial_number2 = ret.serial_number;
+    ret.access_key2 = ret.access_key;
+    ret.name.encode(this->character->disp.name.decode());
+    ret.client_config = this->client_config;
+    this->channel.send(0x9E, 0x01, &ret, extended ? sizeof(ret) : sizeof(C_Login_GC_9E));
+
+  } else if (this->channel.version == Version::XB_V3) {
+    C_LoginExtended_XB_9E ret;
+    ret.player_tag = this->guild_card_number ? 0xFFFF0000 : 0x00010000;
+    ret.guild_card_number = this->guild_card_number;
+    ret.sub_version = default_sub_version_for_version(this->channel.version);
+    ret.is_extended = extended ? 1 : 0;
+    ret.language = this->channel.language;
+    ret.serial_number.encode(this->xb_gamertag);
+    ret.access_key.encode(phosg::string_printf("%016" PRIX64, this->xb_user_id));
+    ret.serial_number2 = ret.serial_number;
+    ret.access_key2 = ret.access_key;
+    ret.name.encode(this->character->disp.name.decode());
+    ret.netloc.internal_ipv4_address = phosg::random_object<uint32_t>();
+    ret.netloc.external_ipv4_address = phosg::random_object<uint32_t>();
+    ret.netloc.port = 9500;
+    phosg::random_data(&ret.netloc.mac_address, sizeof(ret.netloc.mac_address));
+    ret.netloc.sg_ip_address = phosg::random_object<uint32_t>();
+    ret.netloc.spi = phosg::random_object<uint32_t>();
+    ret.netloc.account_id = this->xb_account_id;
+    ret.netloc.unknown_a3.clear(0);
+    ret.xb_user_id_high = this->xb_user_id >> 32;
+    ret.xb_user_id_low = this->xb_user_id;
+    this->channel.send(0x9E, 0x01, &ret, extended ? sizeof(ret) : sizeof(C_Login_DC_PC_GC_9D));
+
+  } else {
+    throw runtime_error("unsupported version");
+  }
+}
+
+void DownloadSession::send_61_98(bool is_98) {
+  uint8_t command = is_98 ? 0x98 : 0x61;
+
+  if (is_v1(this->channel.version)) {
+    C_CharacterData_DCv1_61_98 ret;
+    ret.inventory = this->character->inventory;
+    ret.disp = convert_player_disp_data<PlayerDispDataDCPCV3, PlayerDispDataBB>(this->character->disp, 1, 1);
+    this->channel.send(command, 0x01, ret);
+
+  } else if (this->channel.version == Version::DC_V2) {
+    C_CharacterData_DCv2_61_98 ret;
+    ret.inventory = this->character->inventory;
+    ret.disp = convert_player_disp_data<PlayerDispDataDCPCV3, PlayerDispDataBB>(this->character->disp, 1, 1);
+    ret.records.challenge = this->character->challenge_records;
+    ret.records.battle = this->character->battle_records;
+    ret.choice_search_config = this->character->choice_search_config;
+    this->channel.send(command, 0x02, ret);
+
+  } else if (this->channel.version == Version::PC_V2) {
+    C_CharacterData_PC_61_98 ret;
+    ret.inventory = this->character->inventory;
+    ret.disp = convert_player_disp_data<PlayerDispDataDCPCV3, PlayerDispDataBB>(this->character->disp, 1, 1);
+    ret.records.challenge = this->character->challenge_records;
+    ret.records.battle = this->character->battle_records;
+    ret.choice_search_config = this->character->choice_search_config;
+    this->channel.send(command, 0x02, ret);
+
+  } else if (is_v3(this->channel.version)) {
+    C_CharacterData_V3_61_98 ret;
+    ret.inventory = this->character->inventory;
+    ret.disp = convert_player_disp_data<PlayerDispDataDCPCV3, PlayerDispDataBB>(this->character->disp, 1, 1);
+    ret.records.challenge = this->character->challenge_records;
+    ret.records.battle = this->character->battle_records;
+    ret.choice_search_config = this->character->choice_search_config;
+    ret.info_board.encode(this->character->info_board.decode());
+    this->channel.send(command, 0x03, ret);
+
+  } else if (this->channel.version == Version::BB_V4) {
+    C_CharacterData_BB_61_98 ret;
+    ret.inventory = this->character->inventory;
+    ret.disp = this->character->disp;
+    ret.records.challenge = this->character->challenge_records;
+    ret.records.battle = this->character->battle_records;
+    ret.choice_search_config = this->character->choice_search_config;
+    ret.info_board.encode(this->character->info_board.decode());
+    this->channel.send(command, 0x04, ret);
+
+  } else {
+    throw runtime_error("unsupported version");
+  }
+}
+
+void DownloadSession::on_channel_input(uint16_t command, uint32_t flag, std::string& data) {
+  // TODO: Use the iovec form of print_data here instead of
+  // prepend_command_header (which copies the string)
+  string full_cmd = prepend_command_header(this->channel.version, this->channel.crypt_in.get(), command, flag, data);
+
+  for (size_t z = 0; z < 0x28 && z < data.size(); z++) {
+    this->prev_cmd_data[z] = data[z];
+  }
+
+  switch (command) {
+    case 0x03: {
+      if (this->channel.version != Version::BB_V4) {
+        throw runtime_error("BB server sent non-BB encryption command");
+      }
+      if (!this->bb_key_file) {
+        throw runtime_error("BB encryption requires a key file");
+      }
+      const auto& cmd = check_size_t<S_ServerInitDefault_BB_03_9B>(data, 0xFFFF);
+      this->channel.crypt_in = make_shared<PSOBBEncryption>(*this->bb_key_file, &cmd.server_key[0], sizeof(cmd.server_key));
+      this->channel.crypt_out = make_shared<PSOBBEncryption>(*this->bb_key_file, &cmd.client_key[0], sizeof(cmd.client_key));
+      this->log.info("Enabled BB encryption");
+      throw runtime_error("not yet implemented"); // Send 93
+      break;
+    }
+
+    case 0x02:
+    case 0x17:
+    case 0x91:
+    case 0x9B: {
+      const auto& cmd = check_size_t<S_ServerInitDefault_DC_PC_V3_02_17_91_9B>(data, 0xFFFF);
+      if (uses_v3_encryption(this->channel.version)) {
+        this->channel.crypt_in = make_shared<PSOV3Encryption>(cmd.server_key);
+        this->channel.crypt_out = make_shared<PSOV3Encryption>(cmd.client_key);
+        this->log.info("Enabled V3 encryption (server key %08" PRIX32 ", client key %08" PRIX32 ")",
+            cmd.server_key.load(), cmd.client_key.load());
+      } else if (!uses_v4_encryption(this->channel.version)) {
+        this->channel.crypt_in = make_shared<PSOV2Encryption>(cmd.server_key);
+        this->channel.crypt_out = make_shared<PSOV2Encryption>(cmd.client_key);
+        this->log.info("Enabled V2 encryption (server key %08" PRIX32 ", client key %08" PRIX32 ")",
+            cmd.server_key.load(), cmd.client_key.load());
+      } else {
+        throw runtime_error("BB server sent non-BB encryption command");
+      }
+
+      if (command == 0x02) {
+        bool is_extended = (this->channel.version == Version::XB_V3);
+        this->send_93_9D_9E(is_extended);
+
+      } else {
+        if (is_v1(this->channel.version)) {
+          C_LoginV1_DC_PC_V3_90 ret;
+          ret.serial_number.encode(phosg::string_printf("%08" PRIX32, this->serial_number));
+          ret.access_key.encode(this->access_key);
+          this->channel.send(0x90, 0x00, ret);
+
+        } else if (is_v2(this->channel.version)) {
+          C_Login_DC_PC_V3_9A ret;
+          ret.serial_number.encode(phosg::string_printf("%08" PRIX32, this->serial_number));
+          ret.access_key.encode(this->access_key);
+          ret.player_tag = this->guild_card_number ? 0xFFFF0000 : 0x00010000;
+          ret.guild_card_number = this->guild_card_number;
+          ret.sub_version = default_sub_version_for_version(this->channel.version);
+          ret.serial_number2 = ret.serial_number;
+          ret.access_key2 = ret.access_key;
+          this->channel.send(0x9A, 0x00, ret);
+
+        } else if (this->channel.version == Version::GC_V3) {
+          C_VerifyAccount_V3_DB ret;
+          ret.serial_number.encode(phosg::string_printf("%08" PRIX32, this->serial_number));
+          ret.access_key.encode(this->access_key);
+          ret.sub_version = default_sub_version_for_version(this->channel.version);
+          ret.serial_number2 = ret.serial_number;
+          ret.access_key2 = ret.access_key;
+          ret.password.encode(this->password);
+          this->channel.send(0xDB, 0x00, ret);
+
+        } else if (this->channel.version == Version::XB_V3) {
+          this->send_93_9D_9E(true);
+
+        } else {
+          throw runtime_error("unsupported version");
+        }
+      }
+
+      break;
+    }
+
+    case 0x90:
+    case 0x9A: {
+      if (flag == 1) {
+        if (is_v1(this->channel.version)) {
+          C_RegisterV1_DC_92 ret;
+          ret.sub_version = default_sub_version_for_version(this->channel.version);
+          ret.language = this->channel.language;
+          ret.hardware_id.encode(phosg::string_printf("%08" PRIX32, this->hardware_id));
+          this->channel.send(0x92, 0x00, ret);
+
+        } else if (!is_v4(this->channel.version)) {
+          C_Register_DC_PC_V3_9C ret;
+          ret.sub_version = default_sub_version_for_version(this->channel.version);
+          ret.language = this->channel.language;
+          if (this->channel.version == Version::XB_V3) {
+            ret.serial_number.encode(this->xb_gamertag);
+            ret.access_key.encode(phosg::string_printf("%016" PRIX64, this->xb_user_id));
+            ret.password.encode("xbox-pso");
+          } else {
+            ret.serial_number.encode(phosg::string_printf("%08" PRIX32, this->serial_number));
+            ret.access_key.encode(this->access_key);
+            ret.password.encode(this->password);
+          }
+          this->channel.send(0x9C, 0x00, ret);
+
+        } else {
+          throw runtime_error("unsupported version");
+        }
+
+      } else if (flag == 0 || flag == 2) {
+        this->send_93_9D_9E(true);
+
+      } else {
+        throw runtime_error("login failed");
+      }
+      break;
+    }
+
+    case 0x92:
+    case 0x9C:
+      if (flag == 0) {
+        throw runtime_error("server rejected login credentials");
+      }
+      this->send_93_9D_9E(true);
+      break;
+
+    case 0x9F: {
+      if (is_v1_or_v2(this->channel.version)) {
+        throw runtime_error("invalid command");
+      }
+      this->channel.send(0x9F, 0x00, this->client_config);
+      break;
+    }
+
+    case 0xB2: {
+      C_ExecuteCodeResult_B3 ret;
+      ret.checksum = 0;
+      ret.return_value = 0;
+      this->channel.send(0xB3, 0x00, ret);
+      break;
+    }
+
+    case 0x04: {
+      const auto& cmd = check_size_t<S_UpdateClientConfig_V3_04>(data, 0x08, sizeof(S_UpdateClientConfig_V3_04));
+      if (!is_v1_or_v2(this->channel.version)) {
+        for (size_t z = 0; z < 0x20; z++) {
+          size_t read_index = z + 8;
+          this->client_config[z] = (read_index < data.size()) ? data[read_index] : this->prev_cmd_data[read_index];
+        }
+      }
+      this->guild_card_number = cmd.guild_card_number;
+      if (!this->sent_96) {
+        C_CharSaveInfo_DCv2_PC_V3_BB_96 ret;
+        ret.creation_timestamp = this->character->creation_timestamp;
+        ret.event_counter = this->character->save_count;
+        this->channel.send(0x96, 0x00, ret);
+        this->sent_96 = true;
+      }
+      break;
+    }
+
+    case 0x97:
+      this->channel.send(0xB1, 0x00);
+      break;
+
+    case 0x95:
+      this->send_61_98(false);
+      break;
+
+    case 0xB1:
+      this->channel.send(0x99, 0x00);
+      break;
+
+    case 0x1A:
+    case 0xD5:
+      if (is_v3(this->channel.version)) {
+        this->channel.send(0xD6, 0x00);
+      }
+      break;
+
+    case 0x07:
+    case 0x1F:
+    case 0xA0:
+    case 0xA1: {
+      C_MenuSelection_10_Flag00 ret;
+
+      auto handle_command = [&]<typename CmdT>() {
+        const auto* items = check_size_vec_t<CmdT>(data, flag + 1);
+        size_t item_index;
+        this->log.info("Ship Select menu:");
+        for (item_index = 1; item_index <= flag; item_index++) {
+          const auto& item = items[item_index];
+          auto text = strip_color(item.text.decode());
+          this->log.info("%zu: (%08" PRIX32 " %08" PRIX32 ") %s", item_index, item.menu_id.load(), item.item_id.load(), text.c_str());
+          if (this->ship_menu_selections.count(text)) {
+            break;
+          }
+        }
+        if (item_index > flag) {
+          if (this->interactive) {
+            while (item_index == 0 || item_index > flag) {
+              this->log.info("Choose response index:");
+              string input = phosg::fgets(stdin);
+              item_index = stoul(input, nullptr, 0);
+            }
+          } else {
+            throw runtime_error("unhandled menu selection");
+          }
+        }
+        ret.menu_id = items[item_index].menu_id;
+        ret.item_id = items[item_index].item_id;
+      };
+
+      if (uses_utf16(this->channel.version)) {
+        handle_command.operator()<S_MenuEntry_PC_BB_07_1F>();
+      } else {
+        handle_command.operator()<S_MenuEntry_DC_V3_07_1F>();
+      }
+
+      this->channel.send(0x10, 0x00, ret);
+      break;
+    }
+
+    case 0x01:
+    case 0x11:
+    case 0x60:
+    case 0x62:
+    case 0x68:
+    case 0x69:
+    case 0x6C:
+    case 0x6D:
+    case 0x88:
+    case 0x8A:
+    case 0xB0:
+    case 0xC5:
+    case 0xDA:
+      break;
+
+    case 0x1D:
+      this->channel.send(0x1D, 0x00);
+      break;
+
+    case 0x19: {
+      const auto& cmd = check_size_t<S_Reconnect_19>(data, sizeof(S_Reconnect_19), 0xFFFF);
+
+      sockaddr_storage ss;
+      auto* sin = reinterpret_cast<sockaddr_in*>(&ss);
+      sin->sin_family = AF_INET;
+      sin->sin_addr.s_addr = htonl(cmd.address);
+      sin->sin_port = htons(cmd.port);
+      string netloc_str = phosg::render_sockaddr_storage(ss);
+      this->log.info("Connecting to %s", netloc_str.c_str());
+
+      struct bufferevent* bev = bufferevent_socket_new(this->base.get(), -1, BEV_OPT_CLOSE_ON_FREE | BEV_OPT_DEFER_CALLBACKS);
+      if (!bev) {
+        throw runtime_error(phosg::string_printf("failed to open socket (%d)", EVUTIL_SOCKET_ERROR()));
+      }
+      this->channel.set_bufferevent(bev, 0);
+      this->channel.crypt_in.reset();
+      this->channel.crypt_out.reset();
+
+      if (bufferevent_socket_connect(this->channel.bev.get(), reinterpret_cast<const sockaddr*>(&ss), sizeof(struct sockaddr_in)) != 0) {
+        throw runtime_error(phosg::string_printf("failed to connect (%d)", EVUTIL_SOCKET_ERROR()));
+      }
+
+      break;
+    }
+
+    case 0x83: {
+      const auto* items = check_size_vec_t<S_LobbyListEntry_83>(data, flag, true);
+      this->lobby_menu_items.clear();
+      for (size_t z = 0; z < flag; z++) {
+        this->lobby_menu_items.emplace_back(items[z]);
+      }
+      break;
+    }
+
+    case 0x67: {
+      // Technically we should assign item IDs here, but the server will never
+      // be able to see that we didn't, so we don't bother
+
+      const auto& game_config = this->game_configs[this->current_game_config_index];
+      if (this->channel.version == Version::PC_V2) {
+        C_CreateGame_PC_C1 ret;
+        ret.name.encode(random_name());
+        ret.password.encode(random_name());
+        ret.difficulty = 0;
+        ret.battle_mode = (game_config.mode == GameMode::BATTLE);
+        ret.challenge_mode = (game_config.mode == GameMode::CHALLENGE);
+        ret.episode = 1;
+        this->channel.send(0xC1, 0x00, ret);
+
+      } else if (!is_v4(this->channel.version)) {
+        C_CreateGame_DC_V3_0C_C1_Ep3_EC ret;
+        ret.name.encode(random_name());
+        ret.password.encode(random_name());
+        ret.difficulty = 0;
+        ret.battle_mode = (game_config.mode == GameMode::BATTLE);
+        ret.challenge_mode = (game_config.mode == GameMode::CHALLENGE);
+        if (is_v1(this->channel.version)) {
+          ret.episode = 0;
+        } else if (game_config.episode == Episode::EP1) {
+          ret.episode = 1;
+        } else if (game_config.episode == Episode::EP2) {
+          ret.episode = 2;
+        } else if (game_config.episode == Episode::EP4) {
+          ret.episode = 4;
+        } else {
+          throw std::logic_error("invalid episode");
+        }
+        this->channel.send(is_v1(this->channel.version) ? 0x0C : 0xC1, 0x00, ret);
+
+      } else {
+        C_CreateGame_BB_C1 ret;
+        ret.name.encode(random_name());
+        ret.password.encode(random_name());
+        ret.difficulty = 0;
+        ret.battle_mode = (game_config.mode == GameMode::BATTLE);
+        ret.challenge_mode = (game_config.mode == GameMode::CHALLENGE);
+        if (game_config.episode == Episode::EP1) {
+          ret.episode = 1;
+        } else if (game_config.episode == Episode::EP2) {
+          ret.episode = 2;
+        } else if (game_config.episode == Episode::EP4) {
+          ret.episode = 4;
+        } else {
+          throw std::logic_error("invalid episode");
+        }
+        ret.solo_mode = (game_config.mode == GameMode::SOLO);
+        this->channel.send(is_v1(this->channel.version) ? 0x0C : 0xC1, 0x00, ret);
+      }
+      break;
+    }
+
+    case 0x64: {
+      this->in_game = true;
+      this->bin_complete = false;
+      this->dat_complete = false;
+      for (size_t z = 0; z < this->character->inventory.num_items; z++) {
+        this->character->inventory.items[z].data.id = 0x00010000 + z;
+      }
+
+      if (!is_v1(this->channel.version)) {
+        this->channel.send(0x8A, 0x00);
+      }
+      this->channel.send(0x6F, 0x00);
+      this->send_next_request();
+      break;
+    }
+
+    case 0xA2: {
+      auto handle_command = [&]<typename CmdT>() {
+        const auto* items = check_size_vec_t<CmdT>(data, flag);
+        for (size_t z = 0; z < flag; z++) {
+          const auto& item = items[z];
+          uint64_t request = (static_cast<uint64_t>(item.menu_id) << 32) | static_cast<uint64_t>(item.item_id);
+          if (!this->done_requests.count(request)) {
+            this->log.info("Adding request %016" PRIX64, request);
+            this->pending_requests.emplace(request, item.name.decode());
+          }
+        }
+      };
+
+      if (this->channel.version == Version::PC_V2) {
+        handle_command.operator()<S_QuestMenuEntry_PC_A2_A4>();
+      } else if (this->channel.version == Version::XB_V3) {
+        handle_command.operator()<S_QuestMenuEntry_XB_A2_A4>();
+      } else if (this->channel.version == Version::BB_V4) {
+        handle_command.operator()<S_QuestMenuEntry_BB_A2_A4>();
+      } else {
+        handle_command.operator()<S_QuestMenuEntry_DC_GC_A2_A4>();
+      }
+      this->send_next_request();
+      break;
+    }
+    case 0x44:
+    case 0xA6: {
+      auto handle_command = [&]<typename CmdT>() {
+        const auto& cmd = check_size_t<CmdT>(data, 0xFFFF);
+        string internal_name = cmd.filename.decode();
+        string filtered_name;
+        for (char ch : internal_name) {
+          filtered_name.push_back((isalnum(ch) || (ch == '-') || (ch == '.') || (ch == '_')) ? ch : '_');
+        }
+        string local_filename = phosg::string_printf(
+            "%s/%016" PRIX64 "_%" PRIu64 "_%s_%c_%s",
+            this->output_dir.c_str(),
+            this->current_request,
+            phosg::now(),
+            phosg::name_for_enum(this->channel.version),
+            char_for_language_code(this->channel.language),
+            filtered_name.c_str());
+        this->open_files.emplace(internal_name, OpenFile{.request = this->current_request, .filename = local_filename, .total_size = cmd.file_size, .data = ""});
+      };
+
+      if (is_dc(this->channel.version)) {
+        handle_command.operator()<S_OpenFile_DC_44_A6>();
+      } else if (!is_v4(this->channel.version)) {
+        handle_command.operator()<S_OpenFile_PC_GC_44_A6>();
+      } else {
+        handle_command.operator()<S_OpenFile_BB_44_A6>();
+      }
+      break;
+    }
+    case 0x13:
+    case 0xA7: {
+      const auto& cmd = check_size_t<S_WriteFile_13_A7>(data);
+      string internal_filename = cmd.filename.decode();
+
+      if (!is_v1_or_v2(this->channel.version)) {
+        C_WriteFileConfirmation_V3_BB_13_A7 ret;
+        ret.filename.encode(internal_filename);
+        this->channel.send(command, flag, ret);
+      }
+
+      auto f_it = this->open_files.find(internal_filename.c_str());
+      if (f_it == this->open_files.end()) {
+        this->log.warning("Received data for non-open file %s", internal_filename.c_str());
+        break;
+      }
+      auto& f = this->open_files.at(cmd.filename.decode());
+      size_t block_offset = flag * 0x400;
+      size_t allowed_block_size = (block_offset < f.total_size)
+          ? min<size_t>(f.total_size - block_offset, 0x400)
+          : 0;
+      size_t data_size = min<size_t>(cmd.data_size, allowed_block_size);
+      size_t block_end_offset = block_offset + data_size;
+      if (block_end_offset > f.data.size()) {
+        f.data.resize(block_end_offset);
+      }
+      memcpy(f.data.data() + block_offset, cmd.data.data(), data_size);
+      if (cmd.data_size != 0x400) {
+        phosg::save_file(f.filename, f.data);
+        this->log.info("Wrote file %s (%zu bytes)", f.filename.c_str(), f.data.size());
+        this->open_files.erase(internal_filename);
+        if (phosg::ends_with(internal_filename, ".bin")) {
+          this->bin_complete = true;
+        } else if (phosg::ends_with(internal_filename, ".dat")) {
+          this->dat_complete = true;
+        }
+        if (this->open_files.empty() && this->bin_complete && this->dat_complete) {
+          if (is_v1_or_v2(this->channel.version)) {
+            this->on_request_complete();
+          } else {
+            this->channel.send(0xAC, 0x00);
+          }
+        }
+      }
+      break;
+    }
+    case 0xAC: {
+      if (is_v1_or_v2(this->channel.version)) {
+        throw runtime_error("unsupported version");
+      }
+      this->on_request_complete();
+      break;
+    }
+  }
+}
+
+void DownloadSession::send_next_request() {
+  if (should_request_category_list) {
+    this->log.info("Requesting quest list");
+    this->channel.send(0xA2, 0x00);
+    if (is_v4(this->channel.version)) {
+      this->channel.send(0xA2, 0x01);
+    }
+    this->should_request_category_list = false;
+
+  } else if (!this->pending_requests.empty()) {
+    if (interactive) {
+      const auto& config = this->game_configs[this->current_game_config_index];
+      this->log.info("Items available to expand (mode=%s, episode=%s):", name_for_mode(config.mode), name_for_episode(config.episode));
+      for (const auto& it : this->pending_requests) {
+        this->log.info("%016" PRIX64 ": %s", it.first, it.second.c_str());
+      }
+      this->log.info("Choose item to expand by ID (q to quit; s to skip to next config):");
+      string input = phosg::fgets(stdin);
+      if (input.empty() || (input == "q\n")) {
+        this->channel.disconnect();
+        return;
+      } else if (input == "s\n") {
+        this->pending_requests.clear();
+        this->on_request_complete();
+      } else {
+        this->current_request = stoull(input, nullptr, 16);
+        this->done_requests.emplace(this->current_request);
+        this->pending_requests.erase(this->current_request);
+      }
+
+    } else {
+      auto item_it = this->pending_requests.begin();
+      this->current_request = item_it->first;
+      this->done_requests.emplace(this->current_request);
+      this->pending_requests.erase(item_it);
+      this->log.info("Sending request %016" PRIX64, this->current_request);
+    }
+
+    C_MenuSelection_10_Flag00 cmd;
+    cmd.menu_id = (this->current_request >> 32) & 0xFFFFFFFF;
+    cmd.item_id = this->current_request & 0xFFFFFFFF;
+    this->channel.send(0x10, 0x00, cmd);
+  } else {
+    this->log.info("No pending requests with current parameters");
+    this->on_request_complete();
+  }
+}
+
+void DownloadSession::on_request_complete() {
+  for (const auto& data : this->on_request_complete_commands) {
+    this->channel.send(data);
+  }
+
+  this->send_61_98(true);
+  this->in_game = false;
+
+  const auto& item = this->lobby_menu_items.at(this->lobby_menu_items.size() / 2);
+  C_LobbySelection_84 ret84;
+  ret84.menu_id = item.menu_id;
+  ret84.item_id = item.item_id;
+  this->channel.send(0x84, 0x00, ret84);
+
+  if (this->pending_requests.empty()) {
+    // Advance to next mode/episode combination
+    this->current_game_config_index++;
+    bool v1 = is_v1(this->channel.version);
+    bool v2 = is_v2(this->channel.version);
+    bool v3 = is_v3(this->channel.version);
+    while ((this->current_game_config_index < this->game_configs.size()) &&
+        ((v1 && !this->game_configs[this->current_game_config_index].v1) ||
+            (v2 && !this->game_configs[this->current_game_config_index].v2) ||
+            (v3 && !this->game_configs[this->current_game_config_index].v3))) {
+      this->current_game_config_index++;
+    }
+    if (this->current_game_config_index >= this->game_configs.size()) {
+      this->log.info("All modes complete");
+      this->channel.disconnect();
+    } else {
+      const auto& config = this->game_configs[this->current_game_config_index];
+      this->log.info("Advancing to %s mode in %s", name_for_mode(config.mode), name_for_episode(config.episode));
+      this->should_request_category_list = true;
+    }
+  }
+}
+
+void DownloadSession::dispatch_on_channel_error(Channel& ch, short events) {
+  auto* session = reinterpret_cast<DownloadSession*>(ch.context_obj);
+  session->on_channel_error(events);
+}
+
+void DownloadSession::on_channel_error(short events) {
+  if (events & BEV_EVENT_CONNECTED) {
+    this->log.info("Server channel connected");
+  }
+  if (events & BEV_EVENT_ERROR) {
+    int err = EVUTIL_SOCKET_ERROR();
+    this->log.warning("Error %d (%s) in server stream", err, evutil_socket_error_to_string(err));
+  }
+  if (events & (BEV_EVENT_ERROR | BEV_EVENT_EOF)) {
+    this->log.info("Server endpoint has disconnected");
+    this->channel.disconnect();
+    event_base_loopexit(this->base.get(), nullptr);
+  }
+}
+
+const std::vector<DownloadSession::GameConfig> DownloadSession::game_configs({
+    {.mode = GameMode::NORMAL, .episode = Episode::EP1, .v1 = true, .v2 = true, .v3 = true},
+    {.mode = GameMode::NORMAL, .episode = Episode::EP2, .v1 = false, .v2 = false, .v3 = true},
+    {.mode = GameMode::NORMAL, .episode = Episode::EP4, .v1 = false, .v2 = false, .v3 = false},
+    {.mode = GameMode::BATTLE, .episode = Episode::EP1, .v1 = false, .v2 = true, .v3 = true},
+    {.mode = GameMode::CHALLENGE, .episode = Episode::EP1, .v1 = false, .v2 = true, .v3 = true},
+    {.mode = GameMode::CHALLENGE, .episode = Episode::EP2, .v1 = false, .v2 = false, .v3 = true},
+    {.mode = GameMode::SOLO, .episode = Episode::EP1, .v1 = false, .v2 = false, .v3 = false},
+    {.mode = GameMode::SOLO, .episode = Episode::EP2, .v1 = false, .v2 = false, .v3 = false},
+    {.mode = GameMode::SOLO, .episode = Episode::EP4, .v1 = false, .v2 = false, .v3 = false},
+});

src/DownloadSession.hh

@@ -0,0 +1,110 @@
+#pragma once
+
+#include <event2/event.h>
+
+#include <functional>
+#include <map>
+#include <memory>
+#include <phosg/Filesystem.hh>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "PSOEncryption.hh"
+#include "PSOProtocol.hh"
+#include "ServerState.hh"
+
+class DownloadSession {
+public:
+  DownloadSession(
+      std::shared_ptr<struct event_base> base,
+      const struct sockaddr_storage& remote,
+      const std::string& output_dir,
+      Version version,
+      uint8_t language,
+      std::shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file,
+      uint32_t hardware_id,
+      uint32_t serial_number,
+      const std::string& access_key,
+      const std::string& username,
+      const std::string& password,
+      const std::string& xb_gamertag,
+      uint64_t xb_user_id,
+      uint64_t xb_account_id,
+      std::shared_ptr<PSOBBCharacterFile> character,
+      const std::unordered_set<std::string>& ship_menu_selections,
+      const std::vector<std::string>& on_request_complete_commands,
+      bool interactive,
+      bool show_command_data);
+  DownloadSession(const DownloadSession&) = delete;
+  DownloadSession(DownloadSession&&) = delete;
+  DownloadSession& operator=(const DownloadSession&) = delete;
+  DownloadSession& operator=(DownloadSession&&) = delete;
+  virtual ~DownloadSession() = default;
+
+protected:
+  // Config (must be set by caller)
+  std::string output_dir;
+  std::shared_ptr<const PSOBBEncryption::KeyFile> bb_key_file;
+  uint32_t hardware_id;
+  uint32_t serial_number;
+  std::string access_key;
+  std::string username;
+  std::string password;
+  std::string xb_gamertag;
+  uint64_t xb_user_id;
+  uint64_t xb_account_id;
+  std::shared_ptr<PSOBBCharacterFile> character;
+  std::unordered_set<std::string> ship_menu_selections;
+  std::vector<std::string> on_request_complete_commands;
+  bool interactive;
+
+  // State (set during session)
+  phosg::PrefixedLogger log;
+  std::shared_ptr<struct event_base> base;
+  Channel channel;
+  uint32_t guild_card_number;
+  parray<uint8_t, 0x28> prev_cmd_data;
+  parray<uint8_t, 0x20> client_config;
+  bool sent_96;
+  std::vector<S_LobbyListEntry_83> lobby_menu_items;
+
+  bool should_request_category_list;
+  uint64_t current_request;
+  std::map<uint64_t, std::string> pending_requests;
+  std::unordered_set<uint64_t> done_requests;
+
+  struct OpenFile {
+    uint64_t request;
+    std::string filename;
+    size_t total_size;
+    std::string data;
+  };
+  std::unordered_map<std::string, OpenFile> open_files;
+
+  struct GameConfig {
+    GameMode mode;
+    Episode episode;
+    bool v1;
+    bool v2;
+    bool v3;
+  };
+  static const std::vector<GameConfig> game_configs;
+  size_t current_game_config_index;
+  bool in_game;
+  bool bin_complete;
+  bool dat_complete;
+
+  static void dispatch_on_channel_input(Channel& ch, uint16_t command, uint32_t flag, std::string& msg);
+  static void dispatch_on_channel_error(Channel& ch, short events);
+  void on_channel_input(uint16_t command, uint32_t flag, std::string& msg);
+  void on_channel_error(short events);
+
+  void send_next_request();
+  void on_request_complete();
+
+  void assign_item_ids(uint32_t base_item_id);
+  void send_93_9D_9E(bool extended);
+  void send_61_98(bool is_98);
+};

src/EnemyType.hh

@@ -1,148 +1,150 @@
-#pragma once
-
-#include <inttypes.h>
-
-#include <phosg/Tools.hh>
-
-#include "StaticGameData.hh"
-
-enum class EnemyType {
-  UNKNOWN = -1,
-  NONE = 0,
-  NON_ENEMY_NPC,
-  AL_RAPPY,
-  ASTARK,
-  BA_BOOTA,
-  BARBA_RAY,
-  BARBAROUS_WOLF,
-  BEE_L,
-  BEE_R,
-  BOOMA,
-  BOOTA,
-  BULCLAW,
-  CANADINE,
-  CANADINE_GROUP,
-  CANANE,
-  CHAOS_BRINGER,
-  CHAOS_SORCERER,
-  CLAW,
-  DARK_BELRA,
-  DARK_FALZ_1,
-  DARK_FALZ_2,
-  DARK_FALZ_3,
-  DARK_GUNNER,
-  DARVANT,
-  DARVANT_ULTIMATE,
-  DE_ROL_LE,
-  DE_ROL_LE_BODY,
-  DE_ROL_LE_MINE,
-  DEATH_GUNNER,
-  DEL_LILY,
-  DEL_RAPPY,
-  DEL_RAPPY_ALT,
-  DELBITER,
-  DELDEPTH,
-  DELSABER,
-  DIMENIAN,
-  DOLMDARL,
-  DOLMOLM,
-  DORPHON,
-  DORPHON_ECLAIR,
-  DRAGON,
-  DUBCHIC,
-  DUBWITCH, // Has no entry in battle params
-  EGG_RAPPY,
-  EPSIGUARD,
-  EPSILON,
-  EVIL_SHARK,
-  GAEL,
-  GAL_GRYPHON,
-  GARANZ,
-  GEE,
-  GI_GUE,
-  GIBBLES,
-  GIGOBOOMA,
-  GILLCHIC,
-  GIRTABLULU,
-  GOBOOMA,
-  GOL_DRAGON,
-  GORAN,
-  GORAN_DETONATOR,
-  GRASS_ASSASSIN,
-  GUIL_SHARK,
-  HALLO_RAPPY,
-  HIDOOM,
-  HILDEBEAR,
-  HILDEBLUE,
-  ILL_GILL,
-  KONDRIEU,
-  LA_DIMENIAN,
-  LOVE_RAPPY,
-  MERICAROL,
-  MERICUS,
-  MERIKLE,
-  MERILLIA,
-  MERILTAS,
-  MERISSA_A,
-  MERISSA_AA,
-  MIGIUM,
-  MONEST,
-  MORFOS,
-  MOTHMANT,
-  NANO_DRAGON,
-  NAR_LILY,
-  OLGA_FLOW_1,
-  OLGA_FLOW_2,
-  PAL_SHARK,
-  PAN_ARMS,
-  PAZUZU,
-  PAZUZU_ALT,
-  PIG_RAY,
-  POFUILLY_SLIME,
-  POUILLY_SLIME,
-  POISON_LILY,
-  PYRO_GORAN,
-  RAG_RAPPY,
-  RECOBOX,
-  RECON,
-  SAINT_MILLION,
-  SAINT_RAPPY,
-  SAND_RAPPY,
-  SAND_RAPPY_ALT,
-  SATELLITE_LIZARD,
-  SATELLITE_LIZARD_ALT,
-  SAVAGE_WOLF,
-  SHAMBERTIN,
-  SINOW_BEAT,
-  SINOW_BERILL,
-  SINOW_GOLD,
-  SINOW_SPIGELL,
-  SINOW_ZELE,
-  SINOW_ZOA,
-  SO_DIMENIAN,
-  UL_GIBBON,
-  VOL_OPT_1,
-  VOL_OPT_2,
-  VOL_OPT_AMP,
-  VOL_OPT_CORE,
-  VOL_OPT_MONITOR,
-  VOL_OPT_PILLAR,
-  YOWIE,
-  YOWIE_ALT,
-  ZE_BOOTA,
-  ZOL_GIBBON,
-  ZU,
-  ZU_ALT,
-  MAX_ENEMY_TYPE,
-};
-
-template <>
-const char* name_for_enum<EnemyType>(EnemyType type);
-template <>
-EnemyType enum_for_name<EnemyType>(const char* name);
-
-bool enemy_type_valid_for_episode(Episode episode, EnemyType enemy_type);
-uint8_t battle_param_index_for_enemy_type(Episode episode, EnemyType enemy_type);
-uint8_t rare_table_index_for_enemy_type(EnemyType enemy_type);
-const std::vector<EnemyType>& enemy_types_for_rare_table_index(Episode episode, uint8_t rt_index);
-bool enemy_type_is_rare(EnemyType type);
+#pragma once
+
+#include <inttypes.h>
+
+#include <phosg/Tools.hh>
+
+#include "StaticGameData.hh"
+#include "Types.hh"
+
+enum class EnemyType {
+  UNKNOWN = -1,
+  NONE = 0,
+  NON_ENEMY_NPC,
+  AL_RAPPY,
+  ASTARK,
+  BA_BOOTA,
+  BARBA_RAY,
+  BARBAROUS_WOLF,
+  BEE_L,
+  BEE_R,
+  BOOMA,
+  BOOTA,
+  BULCLAW,
+  BULK,
+  CANADINE,
+  CANADINE_GROUP,
+  CANANE,
+  CHAOS_BRINGER,
+  CHAOS_SORCERER,
+  CLAW,
+  DARK_BELRA,
+  DARK_FALZ_1,
+  DARK_FALZ_2,
+  DARK_FALZ_3,
+  DARK_GUNNER,
+  DARVANT,
+  DARVANT_ULTIMATE,
+  DE_ROL_LE,
+  DE_ROL_LE_BODY,
+  DE_ROL_LE_MINE,
+  DEATH_GUNNER,
+  DEL_LILY,
+  DEL_RAPPY,
+  DEL_RAPPY_ALT,
+  DELBITER,
+  DELDEPTH,
+  DELSABER,
+  DIMENIAN,
+  DOLMDARL,
+  DOLMOLM,
+  DORPHON,
+  DORPHON_ECLAIR,
+  DRAGON,
+  DUBCHIC,
+  DUBWITCH, // Has no entry in battle params
+  EGG_RAPPY,
+  EPSIGUARD,
+  EPSILON,
+  EVIL_SHARK,
+  GAEL,
+  GAL_GRYPHON,
+  GARANZ,
+  GEE,
+  GI_GUE,
+  GIBBLES,
+  GIGOBOOMA,
+  GILLCHIC,
+  GIRTABLULU,
+  GOBOOMA,
+  GOL_DRAGON,
+  GORAN,
+  GORAN_DETONATOR,
+  GRASS_ASSASSIN,
+  GUIL_SHARK,
+  HALLO_RAPPY,
+  HIDOOM,
+  HILDEBEAR,
+  HILDEBLUE,
+  ILL_GILL,
+  KONDRIEU,
+  LA_DIMENIAN,
+  LOVE_RAPPY,
+  MERICAROL,
+  MERICUS,
+  MERIKLE,
+  MERILLIA,
+  MERILTAS,
+  MERISSA_A,
+  MERISSA_AA,
+  MIGIUM,
+  MONEST,
+  MORFOS,
+  MOTHMANT,
+  NANO_DRAGON,
+  NAR_LILY,
+  OLGA_FLOW_1,
+  OLGA_FLOW_2,
+  PAL_SHARK,
+  PAN_ARMS,
+  PAZUZU,
+  PAZUZU_ALT,
+  PIG_RAY,
+  POFUILLY_SLIME,
+  POUILLY_SLIME,
+  POISON_LILY,
+  PYRO_GORAN,
+  RAG_RAPPY,
+  RECOBOX,
+  RECON,
+  SAINT_MILLION,
+  SAINT_RAPPY,
+  SAND_RAPPY,
+  SAND_RAPPY_ALT,
+  SATELLITE_LIZARD,
+  SATELLITE_LIZARD_ALT,
+  SAVAGE_WOLF,
+  SHAMBERTIN,
+  SINOW_BEAT,
+  SINOW_BERILL,
+  SINOW_GOLD,
+  SINOW_SPIGELL,
+  SINOW_ZELE,
+  SINOW_ZOA,
+  SO_DIMENIAN,
+  UL_GIBBON,
+  VOL_OPT_1,
+  VOL_OPT_2,
+  VOL_OPT_AMP,
+  VOL_OPT_CORE,
+  VOL_OPT_MONITOR,
+  VOL_OPT_PILLAR,
+  YOWIE,
+  YOWIE_ALT,
+  ZE_BOOTA,
+  ZOL_GIBBON,
+  ZU,
+  ZU_ALT,
+  MAX_ENEMY_TYPE,
+};
+
+template <>
+const char* phosg::name_for_enum<EnemyType>(EnemyType type);
+template <>
+EnemyType phosg::enum_for_name<EnemyType>(const char* name);
+
+bool enemy_type_valid_for_episode(Episode episode, EnemyType enemy_type);
+uint8_t battle_param_index_for_enemy_type(Episode episode, EnemyType enemy_type);
+uint8_t rare_table_index_for_enemy_type(EnemyType enemy_type);
+const std::vector<EnemyType>& enemy_types_for_rare_table_index(Episode episode, uint8_t rt_index);
+bool enemy_type_is_rare(EnemyType type);

src/Episode3/AssistServer.hh

@@ -39,17 +39,17 @@ private:
 
 public:
   parray<AssistEffect, 4> assist_effects;
-  std::shared_ptr<const CardIndex::CardEntry> assist_card_defs[4];
+  bcarray<std::shared_ptr<const CardIndex::CardEntry>, 4> assist_card_defs;
   uint32_t num_assist_cards_set;
   parray<uint8_t, 4> client_ids_with_assists;
   parray<AssistEffect, 4> active_assist_effects;
-  std::shared_ptr<const CardIndex::CardEntry> active_assist_card_defs[4];
+  bcarray<std::shared_ptr<const CardIndex::CardEntry>, 4> active_assist_card_defs;
   uint32_t num_active_assists;
-  std::shared_ptr<HandAndEquipState> hand_and_equip_states[4];
-  std::shared_ptr<parray<CardShortStatus, 0x10>> card_short_statuses[4];
-  std::shared_ptr<DeckEntry> deck_entries[4];
-  std::shared_ptr<parray<ActionChainWithConds, 9>> set_card_action_chains[4];
-  std::shared_ptr<parray<ActionMetadata, 9>> set_card_action_metadatas[4];
+  bcarray<std::shared_ptr<HandAndEquipState>, 4> hand_and_equip_states;
+  bcarray<std::shared_ptr<parray<CardShortStatus, 0x10>>, 4> card_short_statuses;
+  bcarray<std::shared_ptr<DeckEntry>, 4> deck_entries;
+  bcarray<std::shared_ptr<parray<ActionChainWithConds, 9>>, 4> set_card_action_chains;
+  bcarray<std::shared_ptr<parray<ActionMetadata, 9>>, 4> set_card_action_metadatas;
 };
 
 } // namespace Episode3

src/Episode3/BattleRecord.cc

@@ -9,7 +9,13 @@ using namespace std;
 
 namespace Episode3 {
 
-BattleRecord::Event::Event(StringReader& r) {
+void BattleRecord::PlayerEntry::print(FILE* stream) const {
+  // TODO: Format this nicely somehow. Maybe factor out the functions in
+  // QuestScript that format some of these structures
+  phosg::print_data(stream, this, sizeof(*this), 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+}
+
+BattleRecord::Event::Event(phosg::StringReader& r) {
   this->type = r.get<Event::Type>();
   this->timestamp = r.get_u64l();
   switch (this->type) {
@@ -32,6 +38,7 @@ BattleRecord::Event::Event(StringReader& r) {
     case Event::Type::GAME_COMMAND:
     case Event::Type::BATTLE_COMMAND:
     case Event::Type::EP3_GAME_COMMAND:
+    case Event::Type::SERVER_DATA_COMMAND:
       this->data = r.read(r.get_u16l());
       break;
     default:
@@ -39,7 +46,7 @@ BattleRecord::Event::Event(StringReader& r) {
   }
 }
 
-void BattleRecord::Event::serialize(StringWriter& w) const {
+void BattleRecord::Event::serialize(phosg::StringWriter& w) const {
   w.put(this->type);
   w.put_u64l(this->timestamp);
   switch (this->type) {
@@ -64,6 +71,7 @@ void BattleRecord::Event::serialize(StringWriter& w) const {
     case Event::Type::GAME_COMMAND:
     case Event::Type::BATTLE_COMMAND:
     case Event::Type::EP3_GAME_COMMAND:
+    case Event::Type::SERVER_DATA_COMMAND:
       w.put_u16l(this->data.size());
       w.write(this->data);
       break;
@@ -72,6 +80,52 @@ void BattleRecord::Event::serialize(StringWriter& w) const {
   }
 }
 
+void BattleRecord::Event::print(FILE* stream) const {
+  string time_str = phosg::format_time(this->timestamp);
+  fprintf(stream, "Event @%016" PRIX64 " (%s) ", this->timestamp, time_str.c_str());
+  switch (this->type) {
+    case Type::PLAYER_JOIN:
+      fprintf(stream, "PLAYER_JOIN %02" PRIX32 "\n", this->players[0].lobby_data.client_id.load());
+      this->players[0].print(stream);
+      break;
+    case Type::PLAYER_LEAVE:
+      fprintf(stream, "PLAYER_LEAVE %02hhu\n", this->leaving_client_id);
+      break;
+    case Type::SET_INITIAL_PLAYERS:
+      fprintf(stream, "SET_INITIAL_PLAYERS");
+      for (const auto& player : this->players) {
+        fprintf(stream, " %02" PRIX32, player.lobby_data.client_id.load());
+      }
+      fputc('\n', stream);
+      for (const auto& player : this->players) {
+        player.print(stream);
+      }
+      break;
+    case Type::BATTLE_COMMAND:
+      fprintf(stream, "BATTLE_COMMAND\n");
+      phosg::print_data(stream, this->data, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+      break;
+    case Type::GAME_COMMAND:
+      fprintf(stream, "GAME_COMMAND\n");
+      phosg::print_data(stream, this->data, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+      break;
+    case Type::EP3_GAME_COMMAND:
+      fprintf(stream, "EP3_GAME_COMMAND\n");
+      phosg::print_data(stream, this->data, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+      break;
+    case Type::CHAT_MESSAGE:
+      fprintf(stream, "CHAT_MESSAGE %08" PRIX32 "\n", this->guild_card_number);
+      phosg::print_data(stream, this->data, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+      break;
+    case Type::SERVER_DATA_COMMAND:
+      fprintf(stream, "SERVER_DATA_COMMAND\n");
+      phosg::print_data(stream, this->data, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
+      break;
+    default:
+      throw runtime_error("unknown event type in battle record");
+  }
+}
+
 BattleRecord::BattleRecord(uint32_t behavior_flags)
     : is_writable(true),
       behavior_flags(behavior_flags),
@@ -83,26 +137,37 @@ BattleRecord::BattleRecord(const string& data)
       behavior_flags(0),
       battle_start_timestamp(0),
       battle_end_timestamp(0) {
-  StringReader r(data);
+  phosg::StringReader r(data);
+
   uint64_t signature = r.get_u64l();
-  if (signature != this->SIGNATURE) {
+  bool has_random_stream;
+  if (signature == this->SIGNATURE_V1) {
+    has_random_stream = false;
+  } else if (signature == this->SIGNATURE_V2) {
+    has_random_stream = true;
+  } else {
     throw runtime_error("incorrect battle record signature");
   }
 
   this->battle_start_timestamp = r.get_u64l();
   this->battle_end_timestamp = r.get_u64l();
   this->behavior_flags = r.get_u32l();
+  if (has_random_stream) {
+    this->random_stream = r.read(r.get_u32l());
+  }
   while (!r.eof()) {
     this->events.emplace_back(r);
   }
 }
 
 string BattleRecord::serialize() const {
-  StringWriter w;
-  w.put_u64l(this->SIGNATURE);
+  phosg::StringWriter w;
+  w.put_u64l(this->SIGNATURE_V2);
   w.put_u64l(this->battle_start_timestamp);
   w.put_u64l(this->battle_end_timestamp);
   w.put_u32l(this->behavior_flags);
+  w.put_u32l(this->random_stream.size());
+  w.write(this->random_stream);
   for (const auto& ev : this->events) {
     ev.serialize(w);
   }
@@ -137,7 +202,7 @@ void BattleRecord::add_player(
   }
   Event& ev = this->events.emplace_back();
   ev.type = Event::Type::PLAYER_JOIN;
-  ev.timestamp = now();
+  ev.timestamp = phosg::now();
   auto& player = ev.players.emplace_back();
   player.lobby_data = lobby_data;
   player.inventory = inventory;
@@ -151,7 +216,7 @@ void BattleRecord::delete_player(uint8_t client_id) {
   }
   Event& ev = this->events.emplace_back();
   ev.type = Event::Type::PLAYER_LEAVE;
-  ev.timestamp = now();
+  ev.timestamp = phosg::now();
   ev.leaving_client_id = client_id;
 }
 
@@ -161,7 +226,7 @@ void BattleRecord::add_command(Event::Type type, const void* data, size_t size)
   }
   Event& ev = this->events.emplace_back();
   ev.type = type;
-  ev.timestamp = now();
+  ev.timestamp = phosg::now();
   ev.data.assign(reinterpret_cast<const char*>(data), size);
 }
 
@@ -171,7 +236,7 @@ void BattleRecord::add_command(Event::Type type, string&& data) {
   }
   Event& ev = this->events.emplace_back();
   ev.type = type;
-  ev.timestamp = now();
+  ev.timestamp = phosg::now();
   ev.data = std::move(data);
 }
 
@@ -182,11 +247,29 @@ void BattleRecord::add_chat_message(
   }
   Event& ev = this->events.emplace_back();
   ev.type = Event::Type::CHAT_MESSAGE;
-  ev.timestamp = now();
+  ev.timestamp = phosg::now();
   ev.guild_card_number = guild_card_number;
   ev.data = std::move(data);
 }
 
+void BattleRecord::add_random_data(const void* data, size_t size) {
+  this->random_stream.append(reinterpret_cast<const char*>(data), size);
+}
+
+vector<string> BattleRecord::get_all_server_data_commands() const {
+  vector<string> ret;
+  for (const auto& event : this->events) {
+    if (event.type == Event::Type::SERVER_DATA_COMMAND) {
+      ret.emplace_back(event.data);
+    }
+  }
+  return ret;
+}
+
+const string& BattleRecord::get_random_stream() const {
+  return this->random_stream;
+}
+
 bool BattleRecord::is_map_definition_event(const Event& ev) {
   if (ev.type == Event::Type::BATTLE_COMMAND) {
     auto& header = check_size_t<G_CardBattleCommandHeader>(ev.data, 0xFFFF);
@@ -204,7 +287,7 @@ void BattleRecord::set_battle_start_timestamp() {
   if (this->battle_start_timestamp != 0) {
     throw logic_error("battle start timestamp is already set");
   }
-  this->battle_start_timestamp = now();
+  this->battle_start_timestamp = phosg::now();
 
   // First, find the correct map definition subcommand to keep, and execute
   // player join/leave events to get the present players
@@ -263,15 +346,33 @@ void BattleRecord::set_battle_start_timestamp() {
     }
   }
   for (; it != this->events.end(); it++) {
-    if (it->type == Event::Type::BATTLE_COMMAND) {
+    if ((it->type == Event::Type::BATTLE_COMMAND) || (it->type == Event::Type::SERVER_DATA_COMMAND)) {
       new_events.emplace_back(std::move(*it));
     }
   }
   this->events = std::move(new_events);
+
+  // Clear any existing random data (there shouldn't be any)
+  this->random_stream.clear();
 }
 
 void BattleRecord::set_battle_end_timestamp() {
-  this->battle_end_timestamp = now();
+  this->battle_end_timestamp = phosg::now();
+}
+
+void BattleRecord::print(FILE* stream) const {
+  string start_str = phosg::format_time(this->battle_start_timestamp);
+  string end_str = phosg::format_time(this->battle_end_timestamp);
+  fprintf(stream, "BattleRecord %s behavior_flags=%08" PRIX32 " start=%016" PRIX64 " (%s) end=%016" PRIX64 " (%s); %zu events\n",
+      this->is_writable ? "writable" : "read-only",
+      this->behavior_flags,
+      this->battle_start_timestamp,
+      start_str.c_str(),
+      this->battle_end_timestamp,
+      end_str.c_str(), this->events.size());
+  for (const auto& event : this->events) {
+    event.print(stream);
+  }
 }
 
 BattleRecordPlayer::BattleRecordPlayer(
@@ -287,19 +388,18 @@ shared_ptr<const BattleRecord> BattleRecordPlayer::get_record() const {
   return this->record;
 }
 
-void BattleRecordPlayer::set_lobby(std::shared_ptr<Lobby> l) {
+void BattleRecordPlayer::set_lobby(shared_ptr<Lobby> l) {
   this->lobby = l;
 }
 
 void BattleRecordPlayer::start() {
   if (this->play_start_timestamp == 0) {
-    this->play_start_timestamp = now();
+    this->play_start_timestamp = phosg::now();
     this->schedule_events();
   }
 }
 
-void BattleRecordPlayer::dispatch_schedule_events(
-    evutil_socket_t, short, void* ctx) {
+void BattleRecordPlayer::dispatch_schedule_events(evutil_socket_t, short, void* ctx) {
   reinterpret_cast<BattleRecordPlayer*>(ctx)->schedule_events();
 }
 
@@ -312,7 +412,7 @@ void BattleRecordPlayer::schedule_events() {
   }
 
   for (;;) {
-    uint64_t relative_ts = now() - this->play_start_timestamp + this->record->battle_start_timestamp;
+    uint64_t relative_ts = phosg::now() - this->play_start_timestamp + this->record->battle_start_timestamp;
 
     if (this->event_it == this->record->events.end()) {
       if (relative_ts >= this->record->battle_end_timestamp) {
@@ -325,7 +425,7 @@ void BattleRecordPlayer::schedule_events() {
       } else {
         // There are no more events to play, but the battle has not officially
         // ended yet - reschedule the event for the end time
-        auto tv = usecs_to_timeval(this->record->battle_end_timestamp - relative_ts);
+        auto tv = phosg::usecs_to_timeval(this->record->battle_end_timestamp - relative_ts);
         event_add(this->next_command_ev.get(), &tv);
       }
       break;
@@ -356,13 +456,17 @@ void BattleRecordPlayer::schedule_events() {
           case BattleRecord::Event::Type::CHAT_MESSAGE:
             send_prepared_chat_message(l, ev.guild_card_number, ev.data);
             break;
+          case BattleRecord::Event::Type::SERVER_DATA_COMMAND:
+            // These are not replayed, since the battle record also contains
+            // the results of these commands.
+            break;
         }
         this->event_it++;
 
       } else {
         // The next event should not occur yet, so reschedule for the time when
         // it should occur
-        auto tv = usecs_to_timeval(this->event_it->timestamp - relative_ts);
+        auto tv = phosg::usecs_to_timeval(this->event_it->timestamp - relative_ts);
         event_add(this->next_command_ev.get(), &tv);
         break;
       }

src/Episode3/BattleRecord.hh

@@ -24,7 +24,9 @@ public:
     PlayerInventory inventory;
     PlayerDispDataDCPCV3 disp;
     le_uint32_t level;
-  } __attribute__((packed));
+
+    void print(FILE* stream) const;
+  } __packed_ws__(PlayerEntry, 0x440);
 
   struct Event {
     enum class Type : uint8_t {
@@ -35,6 +37,7 @@ public:
       GAME_COMMAND = 4,
       EP3_GAME_COMMAND = 5,
       CHAT_MESSAGE = 6,
+      SERVER_DATA_COMMAND = 7,
     };
 
     // Fields used for all events
@@ -50,8 +53,9 @@ public:
     std::string data;
 
     Event() = default;
-    explicit Event(StringReader& r);
-    void serialize(StringWriter& w) const;
+    explicit Event(phosg::StringReader& r);
+    void serialize(phosg::StringWriter& w) const;
+    void print(FILE* stream) const;
   };
 
   explicit BattleRecord(uint32_t behavior_flags);
@@ -72,6 +76,7 @@ public:
   void add_command(Event::Type type, const void* data, size_t size);
   void add_command(Event::Type type, std::string&& data);
   void add_chat_message(uint32_t guild_card_number, std::string&& data);
+  void add_random_data(const void* data, size_t size);
   // This function collapses all the existing player join/leave events into a
   // single SET_INITIAL_PLAYERS event, and deletes all events before the latest
   // BATTLE_COMMAND command that specifies the battle map. This should provide a
@@ -79,8 +84,14 @@ public:
   void set_battle_start_timestamp();
   void set_battle_end_timestamp();
 
+  void print(FILE* stream) const;
+
+  std::vector<std::string> get_all_server_data_commands() const;
+  const std::string& get_random_stream() const;
+
 private:
-  static constexpr uint64_t SIGNATURE = 0x14C946D56D1DAC50;
+  static constexpr uint64_t SIGNATURE_V1 = 0x14C946D56D1DAC50;
+  static constexpr uint64_t SIGNATURE_V2 = 0xD01E5EC12853C377;
 
   static bool is_map_definition_event(const Event& ev);
 
@@ -90,15 +101,14 @@ private:
   uint64_t battle_start_timestamp;
   uint64_t battle_end_timestamp;
   std::deque<Event> events;
+  std::string random_stream;
 
   friend class BattleRecordPlayer;
 };
 
 class BattleRecordPlayer {
 public:
-  BattleRecordPlayer(
-      std::shared_ptr<const BattleRecord> rec,
-      std::shared_ptr<struct event_base> base);
+  BattleRecordPlayer(std::shared_ptr<const BattleRecord> rec, std::shared_ptr<struct event_base> base);
   ~BattleRecordPlayer() = default;
 
   std::shared_ptr<const BattleRecord> get_record() const;
@@ -116,6 +126,7 @@ private:
   std::shared_ptr<struct event_base> base;
   std::weak_ptr<Lobby> lobby;
   std::shared_ptr<struct event> next_command_ev;
+  phosg::StringReader random_r;
 };
 
 } // namespace Episode3

src/Episode3/Card.cc

@@ -123,7 +123,7 @@ ssize_t Card::apply_abnormal_condition(
     int8_t dice_roll_value,
     int8_t random_percent) {
   auto s = this->server();
-  auto log = s->log_stack(string_printf("apply_abnormal_condition(%02hhX, @%04X, @%04X, %hd, %hhd, %hhd): ", def_effect_index, target_card_ref, sc_card_ref, value, dice_roll_value, random_percent));
+  auto log = s->log_stack(phosg::string_printf("apply_abnormal_condition(%02hhX, @%04X, @%04X, %hd, %hhd, %hhd): ", def_effect_index, target_card_ref, sc_card_ref, value, dice_roll_value, random_percent));
   bool is_nte = s->options.is_nte();
 
   ssize_t existing_cond_index;
@@ -204,7 +204,7 @@ ssize_t Card::apply_abnormal_condition(
     }
   }
 
-  string cond_str = cond.str();
+  string cond_str = cond.str(s);
   log.debug("wrote condition %zd => %s", cond_index, cond_str.c_str());
 
   if (!is_nte) {
@@ -213,7 +213,7 @@ ssize_t Card::apply_abnormal_condition(
       if (this->action_chain.conditions[z].type == ConditionType::NONE) {
         continue;
       }
-      string cond_str = cond.str();
+      string cond_str = cond.str(s);
       log.debug("sorted conditions: [%zu] => %s", z, cond_str.c_str());
     }
   }
@@ -298,7 +298,7 @@ void Card::commit_attack(
     size_t strike_number,
     int16_t* out_effective_damage) {
   auto s = this->server();
-  auto log = s->log_stack(string_printf("commit_attack(@%04hX #%04hX, @%04hX #%04hX => %hd (str%zu)): ", this->get_card_ref(), this->get_card_id(), attacker_card->get_card_ref(), attacker_card->get_card_id(), damage, strike_number));
+  auto log = s->log_stack(phosg::string_printf("commit_attack(@%04hX #%04hX, @%04hX #%04hX => %hd (str%zu)): ", this->get_card_ref(), this->get_card_id(), attacker_card->get_card_ref(), attacker_card->get_card_id(), damage, strike_number));
   bool is_nte = s->options.is_nte();
 
   int16_t effective_damage = damage;
@@ -396,8 +396,8 @@ int16_t Card::compute_defense_power_for_attacker_card(shared_ptr<const Card> att
     }
   }
 
-  s->card_special->apply_action_conditions(3, attacker_card, this->shared_from_this(), 0x08, nullptr);
-  s->card_special->apply_action_conditions(3, attacker_card, this->shared_from_this(), 0x10, nullptr);
+  s->card_special->apply_action_conditions(EffectWhen::BEFORE_ANY_CARD_ATTACK, attacker_card, this->shared_from_this(), 0x08, nullptr);
+  s->card_special->apply_action_conditions(EffectWhen::BEFORE_ANY_CARD_ATTACK, attacker_card, this->shared_from_this(), 0x10, nullptr);
   return this->action_metadata.defense_power + this->action_metadata.defense_bonus;
 }
 
@@ -507,7 +507,7 @@ void Card::execute_attack(shared_ptr<Card> attacker_card) {
   }
 
   auto s = this->server();
-  auto log = s->log_stack(string_printf("execute_attack(@%04X #%04X, @%04X #%04X): ", this->get_card_ref(), this->get_card_id(), attacker_card->get_card_ref(), attacker_card->get_card_id()));
+  auto log = s->log_stack(phosg::string_printf("execute_attack(@%04X #%04X, @%04X #%04X): ", this->get_card_ref(), this->get_card_id(), attacker_card->get_card_ref(), attacker_card->get_card_id()));
   bool is_nte = s->options.is_nte();
 
   this->card_flags &= 0xFFFFFFF3;
@@ -714,7 +714,7 @@ int32_t Card::move_to_location(const Location& loc) {
         uint8_t trap_type = s->overlay_state.tiles[loc.y][loc.x] & 0x0F;
         uint16_t trap_card_id = s->overlay_state.trap_card_ids_nte[trap_type];
         if (other_ps->replace_assist_card_by_id(trap_card_id)) {
-          G_Unknown_Ep3_6xB4x2C cmd;
+          G_EnqueueAnimation_Ep3_6xB4x2C cmd;
           cmd.change_type = 1;
           cmd.client_id = other_ps->client_id;
           cmd.unknown_a2[0] = trap_card_id;
@@ -728,7 +728,7 @@ int32_t Card::move_to_location(const Location& loc) {
     for (size_t warp_end = 0; warp_end < 2; warp_end++) {
       if ((s->warp_positions[warp_type][warp_end][0] == this->loc.x) &&
           (s->warp_positions[warp_type][warp_end][1] == this->loc.y)) {
-        G_Unknown_Ep3_6xB4x2C cmd;
+        G_EnqueueAnimation_Ep3_6xB4x2C cmd;
         cmd.loc.x = this->loc.x;
         cmd.loc.y = this->loc.y;
         this->loc.x = s->warp_positions[warp_type][warp_end ^ 1][0];
@@ -905,7 +905,7 @@ void Card::clear_action_chain_and_metadata_and_most_flags() {
 
 void Card::compute_action_chain_results(bool apply_action_conditions, bool ignore_this_card_ap_tp) {
   auto s = this->server();
-  auto log = s->log_stack(string_printf("compute_action_chain_results(@%04hX #%04hX): ", this->get_card_ref(), this->get_card_id()));
+  auto log = s->log_stack(phosg::string_printf("compute_action_chain_results(@%04hX #%04hX): ", this->get_card_ref(), this->get_card_id()));
   bool is_nte = s->options.is_nte();
 
   this->action_chain.compute_attack_medium(s);
@@ -914,7 +914,7 @@ void Card::compute_action_chain_results(bool apply_action_conditions, bool ignor
   this->action_chain.chain.tp_effect_bonus = 0;
 
   log.debug("(initial) medium=%s, strike_count=%hhu, ap_effect_bonus=%hhd, tp_effect_bonus=%hhd",
-      name_for_attack_medium(this->action_chain.chain.attack_medium),
+      phosg::name_for_enum(this->action_chain.chain.attack_medium),
       this->action_chain.chain.strike_count,
       this->action_chain.chain.ap_effect_bonus,
       this->action_chain.chain.tp_effect_bonus);
@@ -982,7 +982,7 @@ void Card::compute_action_chain_results(bool apply_action_conditions, bool ignor
 
   if (apply_action_conditions) {
     auto this_sh = this->shared_from_this();
-    s->card_special->apply_action_conditions(3, this_sh, this_sh, 1, nullptr);
+    s->card_special->apply_action_conditions(EffectWhen::BEFORE_ANY_CARD_ATTACK, this_sh, this_sh, 1, nullptr);
     log.debug("applied action conditions (1)");
   } else {
     log.debug("skipped applying action conditions (1)");
@@ -1120,7 +1120,7 @@ void Card::compute_action_chain_results(bool apply_action_conditions, bool ignor
 
   if (apply_action_conditions) {
     auto this_sh = this->shared_from_this();
-    s->card_special->apply_action_conditions(0x03, this_sh, this_sh, 2, nullptr);
+    s->card_special->apply_action_conditions(EffectWhen::BEFORE_ANY_CARD_ATTACK, this_sh, this_sh, 2, nullptr);
     log.debug("applied action conditions (2)");
     if (!is_nte && this->action_chain.check_flag(0x100)) {
       this->action_chain.chain.damage = min<int16_t>(this->action_chain.chain.damage + 5, 99);
@@ -1154,6 +1154,11 @@ void Card::compute_action_chain_results(bool apply_action_conditions, bool ignor
       }
     }
   }
+
+  if (log.should_log(phosg::LogLevel::DEBUG)) {
+    string chain_str = this->action_chain.str(s);
+    log.debug("result computed as %s", chain_str.c_str());
+  }
 }
 
 void Card::unknown_802380C0() {
@@ -1171,7 +1176,7 @@ void Card::unknown_80237F98(bool require_condition_20_or_21) {
   for (ssize_t z = 8; z >= 0; z--) {
     if (this->action_chain.conditions[z].type != ConditionType::NONE) {
       if (!require_condition_20_or_21 ||
-          s->card_special->condition_has_when_20_or_21(this->action_chain.conditions[z])) {
+          s->card_special->condition_applies_on_sc_or_item_attack(this->action_chain.conditions[z])) {
         ActionState as;
         auto& cond = this->action_chain.conditions[z];
         if (!s->card_special->is_card_targeted_by_condition(cond, as, this->shared_from_this())) {
@@ -1219,37 +1224,46 @@ void Card::move_phase_before() {
 }
 
 void Card::unknown_80236374(shared_ptr<Card> other_card, const ActionState* as) {
-  auto log = this->server()->log_stack(string_printf("unknown_80236374(@%04hX #%04hX, @%04hX #%04hX): ", this->get_card_ref(), this->get_card_id(), other_card->get_card_ref(), other_card->get_card_id()));
+  auto s = this->server();
+  auto log = s->log_stack(phosg::string_printf("unknown_80236374(@%04hX #%04hX, @%04hX #%04hX): ", this->get_card_ref(), this->get_card_id(), other_card->get_card_ref(), other_card->get_card_id()));
+
+  if (log.should_log(phosg::LogLevel::DEBUG)) {
+    if (as) {
+      string as_str = as->str(s);
+      log.debug("as = %s", as_str.c_str());
+    } else {
+      log.debug("as = null");
+    }
+  }
+
   auto check_card = [&](shared_ptr<Card> card) -> void {
     if (card) {
       if (!card->unknown_80236554(other_card, as)) {
+        log.debug("check_card @%04hX #%04hX => false", card->get_card_ref(), card->get_card_id());
         card->action_metadata.clear_flags(0x20);
       } else {
+        log.debug("check_card @%04hX #%04hX => true", card->get_card_ref(), card->get_card_id());
         card->action_metadata.set_flags(0x20);
       }
     }
   };
 
   for (size_t client_id = 0; client_id < 4; client_id++) {
-    auto ps = this->server()->player_states[client_id];
-    if (ps) {
-      if (this->server()->get_current_team_turn2() != ps->get_team_id()) {
-        check_card(ps->get_sc_card());
-        for (size_t set_index = 0; set_index < 8; set_index++) {
-          check_card(ps->get_set_card(set_index));
-        }
+    auto ps = s->player_states[client_id];
+    if (ps && (s->get_current_team_turn2() != ps->get_team_id())) {
+      check_card(ps->get_sc_card());
+      for (size_t set_index = 0; set_index < 8; set_index++) {
+        check_card(ps->get_set_card(set_index));
       }
     }
   }
 
   for (size_t client_id = 0; client_id < 4; client_id++) {
-    auto ps = this->server()->player_states[client_id];
-    if (ps) {
-      if (this->server()->get_current_team_turn2() == ps->get_team_id()) {
-        check_card(ps->get_sc_card());
-        for (size_t set_index = 0; set_index < 8; set_index++) {
-          check_card(ps->get_set_card(set_index));
-        }
+    auto ps = s->player_states[client_id];
+    if (ps && (s->get_current_team_turn2() == ps->get_team_id())) {
+      check_card(ps->get_sc_card());
+      for (size_t set_index = 0; set_index < 8; set_index++) {
+        check_card(ps->get_set_card(set_index));
       }
     }
   }
@@ -1361,14 +1375,17 @@ bool Card::is_guard_item() const {
 }
 
 bool Card::unknown_80236554(shared_ptr<Card> other_card, const ActionState* as) {
-  auto log = this->server()->log_stack(other_card
-          ? string_printf("unknown_80236554(@%04hX #%04hX, @%04hX #%04hX): ", this->get_card_ref(), this->get_card_id(), other_card->get_card_ref(), other_card->get_card_id())
-          : string_printf("unknown_80236554(@%04hX #%04hX, null): ", this->get_card_ref(), this->get_card_id()));
-  if (as) {
-    string as_str = as->str();
-    log.debug("as = %s", as_str.c_str());
-  } else {
-    log.debug("as = null");
+  auto s = this->server();
+  auto log = s->log_stack(other_card
+          ? phosg::string_printf("unknown_80236554(@%04hX #%04hX, @%04hX #%04hX): ", this->get_card_ref(), this->get_card_id(), other_card->get_card_ref(), other_card->get_card_id())
+          : phosg::string_printf("unknown_80236554(@%04hX #%04hX, null): ", this->get_card_ref(), this->get_card_id()));
+  if (log.should_log(phosg::LogLevel::DEBUG)) {
+    if (as) {
+      string as_str = as->str(s);
+      log.debug("as = %s", as_str.c_str());
+    } else {
+      log.debug("as = null");
+    }
   }
 
   bool ret = false;
@@ -1403,31 +1420,38 @@ bool Card::unknown_80236554(shared_ptr<Card> other_card, const ActionState* as)
   log.debug("last attack damage stats cleared");
 
   if (other_card) {
-    this->server()->card_special->apply_action_conditions(0x03, other_card, this->shared_from_this(), 0x20, as);
-    this->server()->card_special->apply_action_conditions(0x17, other_card, this->shared_from_this(), 0x40, as);
+    log.debug("applying BEFORE_ANY_CARD_ATTACK conditions");
+    s->card_special->apply_action_conditions(
+        EffectWhen::BEFORE_ANY_CARD_ATTACK, other_card, this->shared_from_this(), 0x20, as);
+    log.debug("applying BEFORE_THIS_CARD_ATTACKED conditions");
+    s->card_special->apply_action_conditions(
+        EffectWhen::BEFORE_THIS_CARD_ATTACKED, other_card, this->shared_from_this(), 0x40, as);
     if (other_card->action_chain.check_flag(0x20000)) {
+      log.debug("attack_bonus cleared due to cancellation");
       this->action_metadata.attack_bonus = 0;
       return ret;
     }
   }
   if (this->card_flags & 2) {
+    log.debug("attack_bonus cleared due to destruction");
     this->action_metadata.attack_bonus = 0;
   }
   return ret;
 }
 
-void Card::unknown_802362D8(shared_ptr<Card> other_card) {
+void Card::execute_attack_on_all_valid_targets(shared_ptr<Card> attacker_card) {
+  auto s = this->server();
   for (size_t client_id = 0; client_id < 4; client_id++) {
-    auto ps = this->server()->player_states[client_id];
+    auto ps = s->player_states[client_id];
     if (ps) {
       shared_ptr<Card> card = ps->get_sc_card();
       if (card) {
-        card->execute_attack(other_card);
+        card->execute_attack(attacker_card);
       }
       for (size_t set_index = 0; set_index < 8; set_index++) {
         shared_ptr<Card> card = ps->get_set_card(set_index);
         if (card) {
-          card->execute_attack(other_card);
+          card->execute_attack(attacker_card);
         }
       }
     }
@@ -1439,7 +1463,7 @@ void Card::apply_attack_result() {
   auto ps = this->player_state();
   bool is_nte = s->options.is_nte();
 
-  auto log = s->log_stack(string_printf("apply_attack_result(@%04hX #%04hX): ", this->get_card_ref(), this->get_card_id()));
+  auto log = s->log_stack(phosg::string_printf("apply_attack_result(@%04hX #%04hX): ", this->get_card_ref(), this->get_card_id()));
   if (!this->action_chain.can_apply_attack()) {
     return;
   }
@@ -1493,7 +1517,7 @@ void Card::apply_attack_result() {
         } else {
           auto target_sc = target_ps->get_sc_card();
           if (!(target_sc->card_flags & 2)) {
-            temp_chain.chain.target_card_refs[temp_chain.chain.target_card_ref_count] = candidate_card->get_card_ref();
+            temp_chain.chain.target_card_refs[temp_chain.chain.target_card_ref_count] = target_sc->get_card_ref();
             temp_chain.chain.target_card_ref_count++;
           }
         }
@@ -1518,7 +1542,7 @@ void Card::apply_attack_result() {
 
     this->compute_action_chain_results(true, false);
     if (!this->action_chain.check_flag(0x40)) {
-      s->card_special->unknown_8024997C(this->shared_from_this());
+      s->card_special->apply_effects_before_attack(this->shared_from_this());
     }
 
     this->compute_action_chain_results(true, false);
@@ -1548,31 +1572,47 @@ void Card::apply_attack_result() {
       }
     }
 
+    if (log.should_log(phosg::LogLevel::DEBUG)) {
+      string as_str = as.str(s);
+      log.debug("as constructed as %s", as_str.c_str());
+    }
+
     for (size_t z = 0; z < this->action_chain.chain.target_card_ref_count; z++) {
       shared_ptr<Card> card = s->card_for_set_card_ref(this->action_chain.chain.target_card_refs[z]);
       if (card) {
         card->current_defense_power = card->action_metadata.attack_bonus;
         if (!this->action_chain.check_flag(0x40)) {
+          log.debug("unknown_8024A6DC(@%04hX #%04hX) ...", card->get_card_ref(), card->get_card_id());
           s->card_special->unknown_8024A6DC(this->shared_from_this(), card);
         }
       }
     }
 
-    this->compute_action_chain_results(1, 0);
+    log.debug("compute_action_chain_results 1 ...");
+    this->compute_action_chain_results(true, false);
+
     if (!this->action_chain.check_flag(0x40)) {
-      s->card_special->unknown_8024997C(this->shared_from_this());
+      log.debug("apply_effects_before_attack ...");
+      s->card_special->apply_effects_before_attack(this->shared_from_this());
     }
     if (!(this->card_flags & 2)) {
-      this->compute_action_chain_results(1, 0);
+      log.debug("compute_action_chain_results 2 ...");
+      this->compute_action_chain_results(true, false);
+      log.debug("check_for_attack_interference ...");
       s->card_special->check_for_attack_interference(this->shared_from_this());
     }
-    this->compute_action_chain_results(1, 0);
+    log.debug("compute_action_chain_results 3 ...");
+    this->compute_action_chain_results(true, false);
+
+    log.debug("unknown_80236374 ...");
     this->unknown_80236374(this->shared_from_this(), nullptr);
-    this->unknown_802362D8(this->shared_from_this());
+    log.debug("execute_attack_on_all_valid_targets ...");
+    this->execute_attack_on_all_valid_targets(this->shared_from_this());
   }
 
   if (!this->action_chain.check_flag(0x40)) {
-    s->card_special->unknown_8024A394(this->shared_from_this());
+    log.debug("apply_effects_after_attack ...");
+    s->card_special->apply_effects_after_attack(this->shared_from_this());
   }
   ps->stats.num_attacks_given++;
 
@@ -1584,6 +1624,7 @@ void Card::apply_attack_result() {
     for (size_t client_id = 0; client_id < 4; client_id++) {
       auto ps = s->player_states[client_id];
       if (ps) {
+        log.debug("unknown_8023C110(%zu) ...", client_id);
         ps->unknown_8023C110();
       }
     }

src/Episode3/Card.hh

@@ -15,11 +15,7 @@ class PlayerState;
 
 class Card : public std::enable_shared_from_this<Card> {
 public:
-  Card(
-      uint16_t card_id,
-      uint16_t card_ref,
-      uint16_t client_id,
-      std::shared_ptr<Server> server);
+  Card(uint16_t card_id, uint16_t card_ref, uint16_t client_id, std::shared_ptr<Server> server);
   void init();
   std::shared_ptr<Server> server();
   std::shared_ptr<const Server> server() const;
@@ -47,8 +43,7 @@ public:
       G_ApplyConditionEffect_Ep3_6xB4x06* cmd,
       size_t strike_number,
       int16_t* out_effective_damage);
-  int16_t compute_defense_power_for_attacker_card(
-      std::shared_ptr<const Card> attacker_card);
+  int16_t compute_defense_power_for_attacker_card(std::shared_ptr<const Card> attacker_card);
   void destroy_set_card(std::shared_ptr<Card> attacker_card);
   int32_t error_code_for_move_to_location(const Location& loc) const;
   void execute_attack(std::shared_ptr<Card> attacker_card);
@@ -73,12 +68,10 @@ public:
   void send_6xB4x4E_4C_4D_if_needed(bool always_send = false);
   void send_6xB4x4E_if_needed(bool always_send = false);
   void set_current_and_max_hp(int16_t hp);
-  void set_current_hp(
-      uint32_t new_hp, bool propagate_shared_hp = true, bool enforce_max_hp = true);
+  void set_current_hp(uint32_t new_hp, bool propagate_shared_hp = true, bool enforce_max_hp = true);
   void update_stats_on_destruction();
   void clear_action_chain_and_metadata_and_most_flags();
-  void compute_action_chain_results(
-      bool apply_action_conditions, bool ignore_this_card_ap_tp);
+  void compute_action_chain_results(bool apply_action_conditions, bool ignore_this_card_ap_tp);
   void unknown_802380C0();
   void unknown_80237F98(bool require_condition_20_or_21);
   void unknown_80237F88();
@@ -92,7 +85,7 @@ public:
   void dice_phase_before();
   bool is_guard_item() const;
   bool unknown_80236554(std::shared_ptr<Card> other_card, const ActionState* as);
-  void unknown_802362D8(std::shared_ptr<Card> other_card);
+  void execute_attack_on_all_valid_targets(std::shared_ptr<Card> attacker_card);
   void apply_attack_result();
 
 private:

src/Episode3/CardSpecial.hh

@@ -6,6 +6,7 @@
 
 #include "../Text.hh"
 #include "DataIndexes.hh"
+#include "Server.hh"
 
 namespace Episode3 {
 
@@ -84,7 +85,7 @@ public:
     /* 84 */ uint32_t target_attack_bonus; // "edm" in expr
     /* 88 */ uint32_t last_attack_preliminary_damage; // "ldm" in expr
     /* 8C */ uint32_t last_attack_damage; // "rdm" in expr
-    /* 90 */ uint32_t total_last_attack_damage; // "fdm" in expr
+    /* 90 */ uint32_t final_last_attack_damage; // "fdm" in expr
     /* 94 */ uint32_t last_attack_damage_count; // "ndm" in expr
     /* 98 */ uint32_t target_current_hp; // "ehp" in expr
     /* 9C */
@@ -94,7 +95,7 @@ public:
     void print(FILE* stream) const;
 
     uint32_t at(size_t index) const;
-  } __attribute__((packed));
+  } __packed_ws__(AttackEnvStats, 0x9C);
 
   CardSpecial(std::shared_ptr<Server> server);
   std::shared_ptr<Server> server();
@@ -105,7 +106,7 @@ public:
   void adjust_dice_boost_if_team_has_condition_52(
       uint8_t team_id, uint8_t* inout_dice_boost, std::shared_ptr<const Card> card);
   void apply_action_conditions(
-      uint8_t when,
+      EffectWhen when,
       std::shared_ptr<const Card> attacker_card,
       std::shared_ptr<Card> defender_card,
       uint32_t flags,
@@ -117,7 +118,7 @@ public:
       uint16_t condition_giver_card_ref,
       uint16_t attacker_card_ref);
   bool apply_defense_condition(
-      uint8_t when,
+      EffectWhen when,
       Condition* defender_cond,
       uint8_t cond_index,
       const ActionState& defense_state,
@@ -126,7 +127,7 @@ public:
       bool unknown_p8);
   bool apply_defense_conditions(
       const ActionState& as,
-      uint8_t when,
+      EffectWhen when,
       std::shared_ptr<Card> defender_card,
       uint32_t flags);
   bool apply_stat_deltas_to_all_cards_from_all_conditions_with_card_ref(
@@ -164,7 +165,7 @@ public:
       uint16_t sc_card_ref);
   StatSwapType compute_stat_swap_type(std::shared_ptr<const Card> card) const;
   void compute_team_dice_bonus(uint8_t team_id);
-  bool condition_has_when_20_or_21(const Condition& cond) const;
+  bool condition_applies_on_sc_or_item_attack(const Condition& cond) const;
   size_t count_action_cards_with_condition_for_all_current_attacks(
       ConditionType cond_type, uint16_t card_ref) const;
   size_t count_action_cards_with_condition_for_current_attack(
@@ -188,7 +189,7 @@ public:
       uint16_t set_card_ref,
       uint16_t sc_card_ref,
       uint8_t random_percent,
-      uint8_t when) const;
+      EffectWhen when) const;
   int32_t evaluate_effect_expr(
       const AttackEnvStats& ast,
       const char* expr,
@@ -275,13 +276,13 @@ public:
       size_t* out_damage_count) const;
   void update_condition_orders(std::shared_ptr<Card> card);
   int16_t max_all_attack_bonuses(size_t* out_count) const;
-  void unknown_80244AA8(std::shared_ptr<Card> card);
+  void apply_effects_after_card_move(std::shared_ptr<Card> card);
   void check_for_defense_interference(
       std::shared_ptr<const Card> attacker_card,
       std::shared_ptr<Card> target_card,
       int16_t* inout_unknown_p4);
   void evaluate_and_apply_effects(
-      uint8_t when,
+      EffectWhen when,
       uint16_t set_card_ref,
       const ActionState& as,
       uint16_t sc_card_ref,
@@ -312,10 +313,10 @@ public:
       std::shared_ptr<const Card> card1,
       const Location& card1_loc,
       std::shared_ptr<const Card> card2) const;
-  void unknown_8024AAB8(const ActionState& as);
+  void apply_effects_after_attack_target_resolution(const ActionState& as);
   void move_phase_before_for_card(std::shared_ptr<Card> unknown_p2);
   void dice_phase_before_for_card(std::shared_ptr<Card> card);
-  template <uint8_t When1, uint8_t When2>
+  template <EffectWhen When1, EffectWhen When2>
   void apply_effects_on_phase_change_t(std::shared_ptr<Card> unknown_p2, const ActionState* existing_as = nullptr);
   void draw_phase_before_for_card(std::shared_ptr<Card> unknown_p2);
   void action_phase_before_for_card(std::shared_ptr<Card> unknown_p2);
@@ -324,10 +325,14 @@ public:
   static std::shared_ptr<Card> sc_card_for_card(std::shared_ptr<Card> unknown_p2);
   void unknown_8024A9D8(const ActionState& pa, uint16_t action_card_ref);
   void check_for_attack_interference(std::shared_ptr<Card> unknown_p2);
-  template <uint8_t When1, uint8_t When2, uint8_t When3, uint8_t When4>
-  void unknown_t2(std::shared_ptr<Card> unknown_p2);
-  void unknown_8024997C(std::shared_ptr<Card> card);
-  void unknown_8024A394(std::shared_ptr<Card> card);
+  template <
+      EffectWhen WhenAllCards,
+      EffectWhen WhenAttackerAndActionCards,
+      EffectWhen WhenAttackerOrHunterSCCard,
+      EffectWhen WhenTargetsAndActionCards>
+  void apply_effects_before_or_after_attack(std::shared_ptr<Card> unknown_p2);
+  void apply_effects_before_attack(std::shared_ptr<Card> card);
+  void apply_effects_after_attack(std::shared_ptr<Card> card);
   bool client_has_atk_dice_boost_condition(uint8_t client_id);
   void unknown_8024A6DC(
       std::shared_ptr<Card> unknown_p2, std::shared_ptr<Card> unknown_p3);

src/Episode3/DataIndexes.hh

@@ -16,6 +16,7 @@
 #include "../PlayerSubordinates.hh"
 #include "../Text.hh"
 #include "../TextIndex.hh"
+#include "../Types.hh"
 
 namespace Episode3 {
 
@@ -37,6 +38,7 @@ enum BehaviorFlag : uint32_t {
   DISABLE_INTERFERENCE = 0x00000100,
   ALLOW_NON_COM_INTERFERENCE = 0x00000200,
   IS_TRIAL_EDITION = 0x00000400,
+  LOG_COMMANDS_IF_LOBBY_MISSING = 0x00000800,
 };
 
 enum class StatSwapType : uint8_t {
@@ -59,8 +61,6 @@ enum class AttackMedium : uint8_t {
   INVALID_FF = 0xFF,
 };
 
-const char* name_for_attack_medium(AttackMedium medium);
-
 enum class CriterionCode : uint8_t {
   NONE = 0x00,
   HU_CLASS_SC = 0x01,
@@ -99,8 +99,6 @@ enum class CriterionCode : uint8_t {
   NON_PHYSICAL_NON_TECH_NON_UNKNOWN_ATTACK_MEDIUM_NON_SC = 0x22,
 };
 
-const char* name_for_criterion_code(CriterionCode code);
-
 enum class CardRank : uint8_t {
   N1 = 0x01,
   R1 = 0x02,
@@ -138,8 +136,6 @@ enum class CardType : uint8_t {
   END_CARD_LIST = 0xFF,
 };
 
-const char* name_for_card_type(CardType type);
-
 enum class CardClass : uint16_t {
   HU_SC = 0x0000,
   RA_SC = 0x0001,
@@ -163,7 +159,6 @@ enum class CardClass : uint16_t {
   ASSIST = 0x0028,
 };
 
-const char* name_for_card_class(CardClass cc);
 bool card_class_is_tech_like(CardClass cc, bool is_nte);
 
 enum class TargetMode : uint8_t {
@@ -179,6 +174,8 @@ enum class TargetMode : uint8_t {
   OWN_FCS = 0x09, // e.g. Traitor
 };
 
+const char* name_for_target_mode(TargetMode target_mode);
+
 enum class ConditionType : uint8_t {
   NONE = 0x00,
   AP_BOOST = 0x01, // Temporarily increase AP by N
@@ -310,7 +307,40 @@ enum class ConditionType : uint8_t {
   ANY_FF = 0xFF, // Used as a wildcard in some search functions
 };
 
-const char* name_for_condition_type(ConditionType cond_type);
+enum class EffectWhen : uint8_t {
+  NONE = 0x00,
+  CARD_SET = 0x01, // Permanent effects like RAMPAGE/PIERCE on SCs, BIG_SWING, AERIAL, etc.; many AC effects also
+  AFTER_ANY_CARD_ATTACK = 0x02, // GIVE_DAMAGE, HEAL, A_H_SWAP_PERM
+  BEFORE_ANY_CARD_ATTACK = 0x03, // AP_LOSS, COMBO_TP
+  BEFORE_DICE_PHASE_THIS_TEAM_TURN = 0x04, // Many different effects
+  CARD_DESTROYED = 0x05, // RETURN_TO_HAND, RETURN, FILIAL, GIVE_OR_TAKE_EXP
+  AFTER_SET_PHASE = 0x06, // Unused
+  BEFORE_MOVE_PHASE = 0x09, // Unused
+  UNKNOWN_0A = 0x0A, // ANTI_ABNORMALITY_2 on Tollaw (non-SC version of another when?)
+  AFTER_ATTACK_TARGET_RESOLUTION = 0x0B, // ABILITY_TRAP via First Attack action card only
+  AFTER_THIS_CARD_ATTACK = 0x0C, // Many effects
+  BEFORE_THIS_CARD_ATTACK = 0x0D, // Conditions, AP_BOOST/TP_BOOST, AP_SILENCE, MULTI_STRIKE
+  BEFORE_ACT_PHASE = 0x0E, // Before act phase (ANTI_ABNORMALITY_2, FIXED_RANGE)
+  BEFORE_DRAW_PHASE = 0x0F, // Unused
+  AFTER_CARD_MOVE = 0x13, // Unused
+  UNKNOWN_15 = 0x15, // Unused
+  AFTER_THIS_CARD_ATTACKED = 0x16, // Conditions, DEATH_COMPANION, GIVE_DAMAGE, AP_GROWTH (Nidra)
+  BEFORE_THIS_CARD_ATTACKED = 0x17, // Defense damage adjustments
+  AFTER_CREATURE_OR_HUNTER_SC_ATTACK = 0x20, // RETURN_TO_HAND, A_T_SWAP_PERM, GIVE_OR_TAKE_EXP
+  BEFORE_CREATURE_OR_HUNTER_SC_ATTACK = 0x21, // Unused
+  UNKNOWN_22 = 0x22, // MISC_AP_BONUSES (SCs only?)
+  BEFORE_MOVE_PHASE_AND_AFTER_CARD_MOVE_FINAL = 0x27, // SET_MV
+  UNKNOWN_29 = 0x29, // MIGHTY_KNUCKLE
+  UNKNOWN_2A = 0x2A, // Unused
+  UNKNOWN_2B = 0x2B, // Unused
+  UNKNOWN_33 = 0x33, // DEF_DISABLE_BY_COST
+  UNKNOWN_34 = 0x34, // Unused
+  UNKNOWN_35 = 0x35, // Unused
+  ATTACK_STAT_OVERRIDES = 0x3D, // BONUS_FROM_LEADER, COPY, ABILITY_TRAP
+  ATTACK_DAMAGE_ADJUSTMENT = 0x3E, // AP_BOOST, SLAYERS_ASSASSINS, WEAK_SPOT_INFLUENCE, GROUP
+  DEFENSE_DAMAGE_ADJUSTMENT = 0x3F, // MOSTLY_HALFGUARDS, ACTION_DISRUPTER
+  BEFORE_DICE_PHASE_ALL_TURNS_FINAL = 0x46, // Pollux Timed Pierce
+};
 
 enum class AssistEffect : uint16_t {
   NONE = 0x0000,
@@ -408,8 +438,6 @@ enum class ActionSubphase : uint8_t {
   INVALID_FF = 0xFF,
 };
 
-const char* name_for_action_subphase(ActionSubphase subphase);
-
 enum class SetupPhase : uint8_t {
   REGISTRATION = 0,
   STARTER_ROLLS = 1,
@@ -439,7 +467,6 @@ enum class Direction : uint8_t {
 Direction turn_left(Direction d);
 Direction turn_right(Direction d);
 Direction turn_around(Direction d);
-const char* name_for_direction(Direction d);
 
 struct Location {
   /* 00 */ uint8_t x;
@@ -458,7 +485,7 @@ struct Location {
 
   void clear();
   void clear_FF();
-} __attribute__((packed));
+} __packed_ws__(Location, 4);
 
 struct CardDefinition {
   struct Stat {
@@ -480,7 +507,8 @@ struct CardDefinition {
 
     void decode_code();
     std::string str() const;
-  } __attribute__((packed));
+    phosg::JSON json() const;
+  } __packed_ws__(Stat, 4);
 
   struct Effect {
     // effect_num is the 1-based index of this effect within the card definition
@@ -495,8 +523,7 @@ struct CardDefinition {
     // rules; for example, the expression "4+4//2" results in 4, not 6.
     /* 02 */ pstring<TextEncoding::ASCII, 0x0F> expr;
     // when specifies in which phase the effect should activate.
-    // TODO: There are many values that can be used here; document them.
-    /* 11 */ uint8_t when;
+    /* 11 */ EffectWhen when;
     // arg1 generally specifies how long the effect activates for.
     /* 12 */ pstring<TextEncoding::ASCII, 4> arg1;
     // arg2 generally specifies a condition for when the effect activates.
@@ -516,10 +543,11 @@ struct CardDefinition {
     bool is_empty() const;
     static std::string str_for_arg(const std::string& arg);
     std::string str(const char* separator = ", ", const TextSet* text_archive = nullptr) const;
-  } __attribute__((packed));
+    phosg::JSON json() const;
+  } __packed_ws__(Effect, 0x20);
 
   /* 0000 */ be_uint32_t card_id;
-  /* 0004 */ parray<uint8_t, 0x40> jp_name;
+  /* 0004 */ pstring<TextEncoding::SJIS, 0x40> jp_name;
 
   // The list of card definitions ends with a "sentinel" definition that isn't a
   // real card, but instead has a negative number in the type field here.
@@ -765,9 +793,9 @@ struct CardDefinition {
   // enormous comment? That's what this array stores.
   /* 009C */ parray<be_uint16_t, 2> drop_rates;
 
-  /* 00A0 */ pstring<TextEncoding::SJIS, 0x14> en_name;
+  /* 00A0 */ pstring<TextEncoding::ISO8859, 0x14> en_name;
   /* 00B4 */ pstring<TextEncoding::SJIS, 0x0B> jp_short_name;
-  /* 00BF */ pstring<TextEncoding::SJIS, 0x08> en_short_name;
+  /* 00BF */ pstring<TextEncoding::ISO8859, 0x08> en_short_name;
   // These effects modify the card's behavior in various situations. Only
   // effects for which effect_num is not zero are used.
   /* 00C7 */ parray<Effect, 3> effects;
@@ -782,7 +810,8 @@ struct CardDefinition {
 
   void decode_range();
   std::string str(bool single_line = true, const TextSet* text_archive = nullptr) const;
-} __attribute__((packed)); // 0x128 bytes in total
+  phosg::JSON json() const;
+} __packed_ws__(CardDefinition, 0x128);
 
 struct CardDefinitionsFooter {
   // Technically the card definitions file is a REL file, so the last 0x20 bytes
@@ -798,10 +827,10 @@ struct CardDefinitionsFooter {
   /* 48 */ be_uint32_t footer_offset;
   /* 4C */ parray<be_uint32_t, 3> unused2;
   /* 58 */
-} __attribute__((packed));
+} __packed_ws__(CardDefinitionsFooter, 0x58);
 
 struct DeckDefinition {
-  /* 00 */ pstring<TextEncoding::SJIS, 0x10> name;
+  /* 00 */ pstring<TextEncoding::MARKED, 0x10> name;
   /* 10 */ be_uint32_t client_id; // 0-3
   // List of card IDs. The card count is the number of nonzero entries here
   // before a zero entry (or 50 if no entries are nonzero). The first card ID is
@@ -818,13 +847,13 @@ struct DeckDefinition {
   /* 82 */ uint8_t second;
   /* 83 */ uint8_t unknown_a2;
   /* 84 */
-} __attribute__((packed));
+} __packed_ws__(DeckDefinition, 0x84);
 
 struct PlayerConfig {
   // The game splits this internally into two structures. The first column of
   // offsets is relative to the start of the first structure; the second column
   // is relative to the start of the second structure.
-  /* 0000:---- */ pstring<TextEncoding::SJIS, 12> rank_text; // From B7 command
+  /* 0000:---- */ pstring<TextEncoding::MARKED, 12> rank_text; // From B7 command
   /* 000C:---- */ parray<uint8_t, 0x1C> unknown_a1;
   /* 0028:---- */ parray<be_uint16_t, 20> tech_menu_shortcut_entries;
   /* 0050:---- */ parray<be_uint32_t, 10> choice_search_config;
@@ -836,7 +865,7 @@ struct PlayerConfig {
   // earlier version, this was the offline records structure, but they later
   // decided to just count online and offline records together in the main
   // records structure and didn't remove the codepath that reads from this.
-  /* 0138:---- */ PlayerRecords_Battle<true> unused_offline_records;
+  /* 0138:---- */ PlayerRecordsBattleBE unused_offline_records;
   /* 0150:---- */ parray<uint8_t, 4> unknown_a4;
   // The PlayerDataSegment structure begins here. In newserv, we combine this
   // structure into PlayerConfig since the two are always used together.
@@ -878,23 +907,25 @@ struct PlayerConfig {
   // card counts array is encrypted in memory most of the time, and they went
   // out of their way to ensure the game uses an area of memory that almost no
   // other game uses, which is also used by the Action Replay.)
-  /* 05A4:0450 */ parray<be_uint64_t, 0x1C2> rare_tokens;
+  /* 05A4:0450 */ parray<be_uint64_t, 450> rare_tokens;
   /* 13B4:1260 */ parray<uint8_t, 0x80> unknown_a7;
   /* 1434:12E0 */ parray<DeckDefinition, 25> decks;
   /* 2118:1FC4 */ parray<uint8_t, 0x08> unknown_a8;
-  /* 2120:1FCC */ be_uint32_t offline_clv_exp; // CLvOff = this / 100
-  /* 2124:1FD0 */ be_uint32_t online_clv_exp; // CLvOn = this / 100
+  /* 2120:1FCC */ be_uint32_t offline_clv_exp; // CLvOff = (this / 100) + 1
+  /* 2124:1FD0 */ be_uint32_t online_clv_exp; // CLvOn = (this / 100) + 1
   struct PlayerReference {
     /* 00 */ be_uint32_t guild_card_number;
-    /* 04 */ pstring<TextEncoding::SJIS, 0x18> player_name;
-  } __attribute__((packed));
-  // This array is updated when a battle is started (via a 6xB4x05 command). The
-  // client adds the opposing players' info to ths first two entries here if the
-  // opponents are human. (The existing entries are always moved back by two
-  // slots, but if one or both opponents are not humans, one or both of the
-  // newly-vacated slots is not filled in.)
+    /* 04 */ pstring<TextEncoding::MARKED, 0x18> name;
+  } __packed_ws__(PlayerReference, 0x1C);
+  // These two arrays are updated when a battle is started (via a 6xB4x05
+  // command). The client adds the opposing players' info to ths first two
+  // entries in recent_human_opponents if the opponents are human. (The
+  // existing entries are always moved back by two slots, but if one or both
+  // opponents are not humans, one or both of the newly-vacated slots is not
+  // filled in.) Both arrays have the most recent entries at the beginning.
   /* 2128:1FD4 */ parray<PlayerReference, 10> recent_human_opponents;
-  /* 2240:20EC */ parray<uint8_t, 0x28> unknown_a10;
+  /* 2240:20EC */ parray<be_uint32_t, 5> recent_battle_start_timestamps;
+  /* 2254:2100 */ parray<uint8_t, 0x14> unknown_a10;
   /* 2268:2114 */ be_uint32_t init_timestamp;
   /* 226C:2118 */ be_uint32_t last_online_battle_start_timestamp;
   // In a certain situation, unknown_t3 is set to init_timestamp plus a multiple
@@ -910,7 +941,42 @@ struct PlayerConfig {
 
   void decrypt();
   void encrypt(uint8_t basis);
-} __attribute__((packed));
+} __packed_ws__(PlayerConfig, 0x2350);
+
+struct PlayerConfigNTE {
+  /* 0000 */ pstring<TextEncoding::MARKED, 12> rank_text;
+  /* 000C */ parray<uint8_t, 0x1C> unknown_a1;
+  /* 0028 */ parray<be_uint16_t, 20> tech_menu_shortcut_entries;
+  /* 0050 */ parray<be_uint32_t, 10> choice_search_config;
+  /* 0078 */ parray<be_uint32_t, 0x10> scenario_progress; // Final has 0x30 entries here
+  /* 00B8 */ PlayerRecordsBattleBE unused_offline_records;
+  /* 00D0 */ parray<uint8_t, 4> unknown_a4;
+  /* 00D4 */ uint8_t is_encrypted;
+  /* 00D5 */ uint8_t basis;
+  /* 00D6 */ parray<uint8_t, 2> unused;
+  /* 00D8 */ parray<uint8_t, 1000> card_counts;
+  /* 04C0 */ parray<be_uint16_t, 50> card_count_checksums;
+  /* 0524 */ parray<be_uint64_t, 300> rare_tokens;
+  /* 0E84 */ parray<DeckDefinition, 25> decks;
+  /* 1B68 */ parray<uint8_t, 0x08> unknown_a8;
+  /* 1B70 */ be_uint32_t offline_clv_exp;
+  /* 1B74 */ be_uint32_t online_clv_exp;
+  /* 1B78 */ parray<PlayerConfig::PlayerReference, 10> recent_human_opponents;
+  /* 1C90 */ parray<be_uint32_t, 5> recent_battle_start_timestamps;
+  /* 1CA4 */ parray<uint8_t, 0x14> unknown_a10;
+  /* 1CB8 */ be_uint32_t init_timestamp;
+  /* 1CBC */ be_uint32_t last_online_battle_start_timestamp;
+  /* 1CC0 */ be_uint32_t unknown_t3;
+  /* 1CC4 */ parray<uint8_t, 0x94> unknown_a14;
+  /* 1D58 */
+
+  PlayerConfigNTE() = default;
+  explicit PlayerConfigNTE(const PlayerConfig& config);
+  operator PlayerConfig() const;
+
+  void decrypt();
+  void encrypt(uint8_t basis);
+} __packed_ws__(PlayerConfigNTE, 0x1D58);
 
 enum class HPType : uint8_t {
   DEFEAT_PLAYER = 0,
@@ -940,8 +1006,8 @@ struct Rules {
   /* 00 */ uint8_t overall_time_limit = 0;
   /* 01 */ uint8_t phase_time_limit = 0; // In seconds; 0 = unlimited
   /* 02 */ AllowedCards allowed_cards = AllowedCards::ALL;
-  /* 03 */ uint8_t min_dice = 1; // 0 = default (1)
-  /* 04 */ uint8_t max_dice = 6; // 0 = default (6)
+  /* 03 */ uint8_t min_dice_value = 1; // 0 = default (1)
+  /* 04 */ uint8_t max_dice_value = 6; // 0 = default (6)
   /* 05 */ uint8_t disable_deck_shuffle = 0; // 0 = shuffle on, 1 = off
   /* 06 */ uint8_t disable_deck_loop = 0; // 0 = loop on, 1 = off
   /* 07 */ uint8_t char_hp = 15;
@@ -952,8 +1018,11 @@ struct Rules {
   /* 0C */ uint8_t disable_dice_boost = 0; // 0 = dice boost on, 1 = off
   // NOTE: The following fields are unused in PSO's implementation, but newserv
   // uses them to implement extended rules.
-  /* 0D */ uint8_t def_dice_range = 0; // High 4 bits = min, low 4 = max
-  /* 0E */ parray<uint8_t, 6> unused;
+  /* 0D */ uint8_t def_dice_value_range = 0; // High 4 bits = min, low 4 = max
+  // These fields specify override dice ranges for the 1-player team in 2v1
+  /* 0E */ uint8_t atk_dice_value_range_2v1 = 0; // High 4 bits = min, low 4 = max
+  /* 0F */ uint8_t def_dice_value_range_2v1 = 0; // High 4 bits = min, low 4 = max
+  /* 10 */ parray<uint8_t, 4> unused;
   /* 14 */
 
   // Annoyingly, this structure is a different size in Episode 3 Trial Edition.
@@ -963,8 +1032,8 @@ struct Rules {
   // likely be more work than it's worth.
 
   Rules() = default;
-  explicit Rules(const JSON& json);
-  JSON json() const;
+  explicit Rules(const phosg::JSON& json);
+  phosg::JSON json() const;
   bool operator==(const Rules& other) const = default;
   bool operator!=(const Rules& other) const = default;
   void clear();
@@ -973,11 +1042,11 @@ struct Rules {
   bool check_invalid_fields() const;
   bool check_and_reset_invalid_fields();
 
-  uint8_t min_def_dice() const;
-  uint8_t max_def_dice() const;
+  std::pair<uint8_t, uint8_t> atk_dice_range(bool is_1p_2v1) const;
+  std::pair<uint8_t, uint8_t> def_dice_range(bool is_1p_2v1) const;
 
   std::string str() const;
-} __attribute__((packed));
+} __packed_ws__(Rules, 0x14);
 
 struct RulesTrial {
   // Most fields here have the same meanings as in the final version.
@@ -1001,7 +1070,7 @@ struct RulesTrial {
   RulesTrial() = default;
   RulesTrial(const Rules&);
   operator Rules() const;
-} __attribute__((packed));
+} __packed_ws__(RulesTrial, 0x0C);
 
 struct StateFlags {
   /* 00 */ le_uint16_t turn_num;
@@ -1023,7 +1092,7 @@ struct StateFlags {
   bool operator!=(const StateFlags& other) const;
   void clear();
   void clear_FF();
-} __attribute__((packed));
+} __packed_ws__(StateFlags, 0x18);
 
 struct MapList {
   be_uint32_t num_maps;
@@ -1051,18 +1120,54 @@ struct MapList {
     /* 021C */ uint8_t map_category;
     /* 021D */ parray<uint8_t, 3> unused;
     /* 0220 */
-  } __attribute__((packed));
+  } __packed_ws__(Entry, 0x220);
 
   // Variable-length fields:
   // Entry entries[num_maps];
   // char strings[...EOF]; // Null-terminated strings, pointed to by offsets in Entry structs
-} __attribute__((packed));
+} __packed_ws__(MapList, 0x10);
 
 struct CompressedMapHeader { // .mnm file format
   le_uint32_t map_number;
   le_uint32_t compressed_data_size;
   // Compressed data immediately follows (which decompresses to a MapDefinition)
-} __attribute__((packed));
+} __packed_ws__(CompressedMapHeader, 8);
+
+struct OverlayState {
+  // In the tiles array, the high 4 bits of each value are the tile type, and
+  // the low 4 bits are the subtype. The types are:
+  // 10: blocked by rock (as if the corresponding map_tiles value was 00)
+  // 20: blocked by fence (as if the corresponding map_tiles value was 00)
+  // 30-34: teleporters (2 of each value may be present)
+  // 40-4F: traps on NTE
+  // 40-44: traps on non-NTE (there may be up to 8 of each type, and one of
+  //   each is chosen to be a real trap at battle start); the trap types are:
+  //   40: Dice Fever, Heavy Fog, Muscular, Immortality, Snail Pace
+  //   41: Gold Rush, Charity, Requiem
+  //   42: Powerless Rain, Trash 1, Empty Hand, Skip Draw
+  //   43: Brave Wind, Homesick, Fly
+  //   44: Dice+1, Battle Royale, Reverse Card, Giant Garden, Fix
+  // 50: blocked by metal box (appears as an improperly-z-buffered teal cube in
+  //   preview; behaves like 10 and 20 in game)
+  // Any other value here will behave like 00 (no special tile behavior).
+  parray<parray<uint8_t, 0x10>, 0x10> tiles;
+
+  // This field appears to be unused in both NTE and the final version. Perhaps
+  // it had some meaning in a pre-NTE version.
+  parray<le_uint32_t, 5> unused1;
+
+  // TODO: Figure out exactly where these colors are used
+  parray<le_uint32_t, 0x10> trap_tile_colors_nte; // Unused on non-NTE
+
+  // This specifies the assist card IDs that each trap value (40-4F) will set
+  // when triggered. This only has an effect on NTE; on non-NTE, this is unused
+  // and a fixed set of assist cards is used instead. (On newserv, the set of
+  // used assist cards can be overridden in the server configuration.)
+  parray<le_uint16_t, 0x10> trap_card_ids_nte;
+
+  OverlayState();
+  void clear();
+} __packed_ws__(OverlayState, 0x174);
 
 struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
   // If tag is not 0x00000100, the game considers the map to be corrupt in
@@ -1159,7 +1264,8 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
     /* 48 */
 
     std::string str() const;
-  } __attribute__((packed));
+    phosg::JSON json() const;
+  } __packed_ws__(CameraSpec, 0x48);
 
   // This array specifies the camera zone maps. A camera zone map is a subset of
   // the main map (specified in map_tiles). Tiles that are part of each camera
@@ -1182,30 +1288,16 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
   // (it is not yet known what the major index represents).
   /* 1AB8 */ parray<parray<CameraSpec, 2>, 3> overview_specs;
 
-  // In the modification_tiles array, the values are:
-  // 10 = blocked by rock (as if the corresponding map_tiles value was 00)
-  // 20 = blocked by fence (as if the corresponding map_tiles value was 00)
-  // 30-34 = teleporters (2 of each value may be present)
-  // 40-4F = traps on NTE
-  // 40-44 = traps on non-NTE (one of each type is chosen at random to be a real
-  //         trap at battle start time)
-  // 50 = blocked by metal box (appears as improperly-z-buffered teal cube in
-  //      preview; behaves like 10 and 20 in game)
-  // The assist cards that each trap type can contain are:
-  //   40: Dice Fever, Heavy Fog, Muscular, Immortality, Snail Pace
-  //   41: Gold Rush, Charity, Requiem
-  //   42: Powerless Rain, Trash 1, Empty Hand, Skip Draw
-  //   43: Brave Wind, Homesick, Fly
-  //   44: Dice+1, Battle Royale, Reverse Card, Giant Garden, Fix
-  /* 1C68 */ parray<parray<uint8_t, 0x10>, 0x10> modification_tiles;
+  // This specifies the locations of blocked tiles, teleporters, and traps. See
+  // the comments in OverlayState for details.
+  /* 1C68 */ OverlayState overlay_state;
 
-  /* 1D68 */ parray<uint8_t, 0x74> unknown_a5;
   /* 1DDC */ Rules default_rules;
 
-  /* 1DF0 */ pstring<TextEncoding::SJIS, 0x14> name;
-  /* 1E04 */ pstring<TextEncoding::SJIS, 0x14> location_name;
-  /* 1E18 */ pstring<TextEncoding::SJIS, 0x3C> quest_name; // == location_name if not a quest
-  /* 1E54 */ pstring<TextEncoding::SJIS, 0x190> description;
+  /* 1DF0 */ pstring<TextEncoding::MARKED, 0x14> name;
+  /* 1E04 */ pstring<TextEncoding::MARKED, 0x14> location_name;
+  /* 1E18 */ pstring<TextEncoding::MARKED, 0x3C> quest_name; // == location_name if not a quest
+  /* 1E54 */ pstring<TextEncoding::MARKED, 0x190> description;
 
   // These fields describe where the map cursor on the preview screen should
   // scroll to
@@ -1213,10 +1305,11 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
   /* 1FE6 */ be_uint16_t map_y;
 
   struct NPCDeck {
-    /* 00 */ pstring<TextEncoding::SJIS, 0x18> name;
+    /* 00 */ pstring<TextEncoding::MARKED, 0x18> deck_name;
     /* 18 */ parray<be_uint16_t, 0x20> card_ids; // Last one appears to always be FFFF
     /* 58 */
-  } __attribute__((packed));
+    phosg::JSON json(uint8_t language) const;
+  } __packed_ws__(NPCDeck, 0x58);
   /* 1FE8 */ parray<NPCDeck, 3> npc_decks; // Unused if name[0] == 0
 
   // These are almost (but not quite) the same format as the entries in
@@ -1227,11 +1320,12 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
     /* 0000 */ parray<be_uint16_t, 2> unknown_a1;
     /* 0004 */ uint8_t is_arkz;
     /* 0005 */ parray<uint8_t, 3> unknown_a2;
-    /* 0008 */ pstring<TextEncoding::SJIS, 0x10> name;
+    /* 0008 */ pstring<TextEncoding::MARKED, 0x10> ai_name;
     // TODO: Figure out exactly how these are used and document here.
     /* 0018 */ parray<be_uint16_t, 0x7E> params;
     /* 0114 */
-  } __attribute__((packed));
+    phosg::JSON json(uint8_t language) const;
+  } __packed_ws__(AIParams, 0x114);
   /* 20F0 */ parray<AIParams, 3> npc_ai_params; // Unused if name[0] == 0
 
   /* 242C */ parray<uint8_t, 8> unknown_a7;
@@ -1261,9 +1355,9 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
   // appears after the battle if it's not blank. dispatch_message appears right
   // before the player chooses a deck if it's not blank; usually it says
   // something like "You can only dispatch <character>".
-  /* 2440 */ pstring<TextEncoding::SJIS, 0x190> before_message;
-  /* 25D0 */ pstring<TextEncoding::SJIS, 0x190> after_message;
-  /* 2760 */ pstring<TextEncoding::SJIS, 0x190> dispatch_message;
+  /* 2440 */ pstring<TextEncoding::MARKED, 0x190> before_message;
+  /* 25D0 */ pstring<TextEncoding::MARKED, 0x190> after_message;
+  /* 2760 */ pstring<TextEncoding::MARKED, 0x190> dispatch_message;
 
   struct DialogueSet {
     // Dialogue sets specify lines that COMs can say at certain points during
@@ -1281,9 +1375,10 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
     /* 0002 */ be_uint16_t percent_chance; // 0-100, or FFFF if unused
     // If the dialogue set activates, the game randomly chooses one of these
     // strings, excluding any that are empty or begin with the character '^'.
-    /* 0004 */ parray<pstring<TextEncoding::SJIS, 0x40>, 4> strings;
+    /* 0004 */ parray<pstring<TextEncoding::MARKED, 0x40>, 4> strings;
     /* 0104 */
-  } __attribute__((packed));
+    phosg::JSON json(uint8_t language) const;
+  } __packed_ws__(DialogueSet, 0x104);
 
   // There are up to 0x10 of these per valid NPC, but only the first 13 of them
   // are used, since each one must have a unique value for .when and the values
@@ -1291,7 +1386,13 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
   /* 28F0 */ parray<parray<DialogueSet, 0x10>, 3> dialogue_sets;
 
   // These card IDs are always given to the player when they win a battle on
-  // this map. Unused entries should be set to FFFF.
+  // this map. Unused entries should be set to FFFF. Cards in this array are
+  // ignored if they have any of these features (in the card definition):
+  // - type is HUNTERS_SC or ARKZ_SC
+  // - card_class is BOSS_ATTACK_ACTION or BOSS_TECH
+  // - rank is D1, D2, or D3
+  // - cannot_drop is 1 (specifically 1; other values don't prevent cards from
+  //   appearing)
   /* 59B0 */ parray<be_uint16_t, 0x10> reward_card_ids;
 
   // These fields are used when determining which cards to drop after the battle
@@ -1312,10 +1413,12 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
   /* 59DC */ uint8_t map_category;
 
   // This field determines block graphics to be used in the Cyber environment.
-  // There are 10 block types (0-9); if this value is > 9, type 0 is used.
+  // There are 10 block types (0-9); if this value is > 9, type 0 is used. This
+  // field has no effect in Ep3 NTE, even though there are 6 different block
+  // texture files on the NTE disc.
   /* 59DD */ uint8_t cyber_block_type;
 
-  /* 59DE */ parray<uint8_t, 2> unknown_a11;
+  /* 59DE */ be_uint16_t unknown_a11;
 
   // This array specifies which SC characters can't participate in the quest
   // (that is, the player is not allowed to choose decks with these SC cards).
@@ -1365,7 +1468,8 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
 
     bool operator==(const EntryState& other) const = default;
     bool operator!=(const EntryState& other) const = default;
-  } __attribute__((packed));
+    phosg::JSON json() const;
+  } __packed_ws__(EntryState, 2);
   /* 5A10 */ parray<EntryState, 4> entry_states;
   /* 5A18 */
 
@@ -1380,7 +1484,8 @@ struct MapDefinition { // .mnmd format; also the format of (decompressed) quests
   void assert_semantically_equivalent(const MapDefinition& other) const;
 
   std::string str(const CardIndex* card_index, uint8_t language) const;
-} __attribute__((packed));
+  phosg::JSON json(uint8_t language) const;
+} __packed_ws__(MapDefinition, 0x5A18);
 
 struct MapDefinitionTrial {
   // This is the format of Episode 3 Trial Edition maps. See the comments in
@@ -1397,22 +1502,21 @@ struct MapDefinitionTrial {
   /* 0118 */ parray<parray<parray<parray<uint8_t, 0x10>, 0x10>, 10>, 2> camera_zone_maps;
   /* 1518 */ parray<parray<MapDefinition::CameraSpec, 10>, 2> camera_zone_specs;
   /* 1AB8 */ parray<parray<MapDefinition::CameraSpec, 2>, 3> overview_specs;
-  /* 1C68 */ parray<parray<uint8_t, 0x10>, 0x10> modification_tiles;
-  /* 1D68 */ parray<uint8_t, 0x74> unknown_a5;
-  /* 1DD4 */ RulesTrial default_rules;
-  /* 1DE8 */ pstring<TextEncoding::SJIS, 0x14> name;
-  /* 1DFC */ pstring<TextEncoding::SJIS, 0x14> location_name;
-  /* 1E10 */ pstring<TextEncoding::SJIS, 0x3C> quest_name;
-  /* 1E4C */ pstring<TextEncoding::SJIS, 0x190> description;
+  /* 1C68 */ OverlayState overlay_state;
+  /* 1DDC */ RulesTrial default_rules;
+  /* 1DE8 */ pstring<TextEncoding::MARKED, 0x14> name;
+  /* 1DFC */ pstring<TextEncoding::MARKED, 0x14> location_name;
+  /* 1E10 */ pstring<TextEncoding::MARKED, 0x3C> quest_name;
+  /* 1E4C */ pstring<TextEncoding::MARKED, 0x190> description;
   /* 1FDC */ be_uint16_t map_x;
   /* 1FDE */ be_uint16_t map_y;
   /* 1FE0 */ parray<MapDefinition::NPCDeck, 3> npc_decks;
   /* 20E8 */ parray<MapDefinition::AIParams, 3> npc_ai_params;
   /* 2424 */ parray<uint8_t, 8> unknown_a7;
   /* 242C */ parray<be_int32_t, 3> npc_ai_params_entry_index;
-  /* 2438 */ pstring<TextEncoding::SJIS, 0x190> before_message;
-  /* 25C8 */ pstring<TextEncoding::SJIS, 0x190> after_message;
-  /* 2758 */ pstring<TextEncoding::SJIS, 0x190> dispatch_message;
+  /* 2438 */ pstring<TextEncoding::MARKED, 0x190> before_message;
+  /* 25C8 */ pstring<TextEncoding::MARKED, 0x190> after_message;
+  /* 2758 */ pstring<TextEncoding::MARKED, 0x190> dispatch_message;
   /* 28E8 */ parray<parray<MapDefinition::DialogueSet, 8>, 3> dialogue_sets;
   /* 4148 */ parray<be_uint16_t, 0x10> reward_card_ids;
   /* 4168 */ be_int32_t win_level_override;
@@ -1421,7 +1525,7 @@ struct MapDefinitionTrial {
   /* 4172 */ be_int16_t field_offset_y;
   /* 4174 */ uint8_t map_category;
   /* 4175 */ uint8_t cyber_block_type;
-  /* 4176 */ parray<uint8_t, 2> unknown_a11;
+  /* 4176 */ be_uint16_t unknown_a11;
   // TODO: This field may contain some version of unavailable_sc_cards and/or
   // entry_states from MapDefinition, but the format isn't the same
   /* 4178 */ parray<uint8_t, 0x28> unknown_t12;
@@ -1429,7 +1533,7 @@ struct MapDefinitionTrial {
 
   MapDefinitionTrial(const MapDefinition& map);
   operator MapDefinition() const;
-} __attribute__((packed));
+} __packed_ws__(MapDefinitionTrial, 0x41A0);
 
 struct COMDeckDefinition {
   size_t index;
@@ -1462,6 +1566,7 @@ public:
   std::shared_ptr<const CardEntry> definition_for_name_normalized(const std::string& name) const;
   std::set<uint32_t> all_ids() const;
   uint64_t definitions_mtime() const;
+  phosg::JSON definitions_json() const;
 
 private:
   static std::string normalize_card_name(const std::string& name);
@@ -1542,14 +1647,37 @@ private:
 
 // TODO: Figure out how to declare these inside the Episode3 namespace.
 template <>
-Episode3::HPType enum_for_name<Episode3::HPType>(const char* name);
+Episode3::HPType phosg::enum_for_name<Episode3::HPType>(const char* name);
 template <>
-const char* name_for_enum<Episode3::HPType>(Episode3::HPType hp_type);
+const char* phosg::name_for_enum<Episode3::HPType>(Episode3::HPType hp_type);
 template <>
-Episode3::DiceExchangeMode enum_for_name<Episode3::DiceExchangeMode>(const char* name);
+Episode3::DiceExchangeMode phosg::enum_for_name<Episode3::DiceExchangeMode>(const char* name);
 template <>
-const char* name_for_enum<Episode3::DiceExchangeMode>(Episode3::DiceExchangeMode dice_exchange_mode);
+const char* phosg::name_for_enum<Episode3::DiceExchangeMode>(Episode3::DiceExchangeMode dice_exchange_mode);
 template <>
-Episode3::AllowedCards enum_for_name<Episode3::AllowedCards>(const char* name);
+Episode3::AllowedCards phosg::enum_for_name<Episode3::AllowedCards>(const char* name);
 template <>
-const char* name_for_enum<Episode3::AllowedCards>(Episode3::AllowedCards allowed_cards);
+const char* phosg::name_for_enum<Episode3::AllowedCards>(Episode3::AllowedCards allowed_cards);
+
+template <>
+const char* phosg::name_for_enum<Episode3::BattlePhase>(Episode3::BattlePhase phase);
+template <>
+const char* phosg::name_for_enum<Episode3::SetupPhase>(Episode3::SetupPhase phase);
+template <>
+const char* phosg::name_for_enum<Episode3::RegistrationPhase>(Episode3::RegistrationPhase phase);
+template <>
+const char* phosg::name_for_enum<Episode3::ActionSubphase>(Episode3::ActionSubphase phase);
+template <>
+const char* phosg::name_for_enum<Episode3::AttackMedium>(Episode3::AttackMedium medium);
+template <>
+const char* phosg::name_for_enum<Episode3::CriterionCode>(Episode3::CriterionCode code);
+template <>
+const char* phosg::name_for_enum<Episode3::CardType>(Episode3::CardType type);
+template <>
+const char* phosg::name_for_enum<Episode3::CardClass>(Episode3::CardClass cc);
+template <>
+const char* phosg::name_for_enum<Episode3::ConditionType>(Episode3::ConditionType cond_type);
+template <>
+const char* phosg::name_for_enum<Episode3::EffectWhen>(Episode3::EffectWhen when);
+template <>
+const char* phosg::name_for_enum<Episode3::Direction>(Episode3::Direction d);

src/Episode3/DeckState.cc

@@ -1,5 +1,7 @@
 #include "DeckState.hh"
 
+#include "Server.hh"
+
 using namespace std;
 
 namespace Episode3 {
@@ -84,43 +86,45 @@ bool DeckState::draw_card_by_ref(uint16_t card_ref) {
   }
 
   uint8_t index = index_for_card_ref(card_ref);
-  if (index > this->entries.size()) {
+  if (index >= this->entries.size()) {
     return false;
   }
 
-  if (this->entries[index].state == CardState::DISCARDED) {
+  auto& entry = this->entries[index];
+  if (entry.state == CardState::DISCARDED) {
     // If the card is discarded, then it should be before the draw index, and we
     // can just change its state.
-    this->entries[index].state = CardState::IN_HAND;
+    entry.state = CardState::IN_HAND;
     return true;
+  }
 
-  } else if (this->entries[index].state == CardState::DRAWABLE) {
-    // If the card is still drawable, we need to move it so it's just in front
-    // of the draw index, then immediately draw it. Ep3 NTE does not handle this
-    // case, but we do even when playing NTE.
-    ssize_t ref_index;
-    for (ref_index = this->card_refs.size(); ref_index >= 0; ref_index--) {
-      if (this->card_refs[ref_index] == card_ref) {
-        break;
-      }
-    }
-    if (ref_index < 0) {
-      return false;
-    }
-
-    size_t ref_uindex = ref_index;
-    for (; ref_uindex > this->draw_index; ref_uindex--) {
-      // Note: draw_index is also unsigned, so ref_uindex cannot be zero here
-      this->card_refs[ref_uindex] = this->card_refs[ref_uindex - 1];
-    }
-    this->card_refs[this->draw_index] = card_ref;
-    this->entries[index].state = CardState::IN_HAND;
-    this->draw_index++;
-    return true;
-
-  } else {
+  if (entry.state != CardState::DRAWABLE) {
     return false;
   }
+
+  // If the card is still drawable, we need to move it so it's just in front of
+  // the draw index, then immediately draw it. Ep3 NTE does not handle this
+  // case, but we do even when playing NTE.
+  size_t ref_index;
+  for (ref_index = 0; ref_index < this->card_refs.size(); ref_index++) {
+    if (this->card_refs[ref_index] == card_ref) {
+      break;
+    }
+  }
+  if (ref_index >= this->card_refs.size()) {
+    return false;
+  }
+
+  for (; ref_index > this->draw_index; ref_index--) {
+    // this->draw_index is also unsigned, so ref_index cannot be zero here
+    this->card_refs[ref_index] = this->card_refs[ref_index - 1];
+  }
+  this->card_refs[this->draw_index] = card_ref;
+
+  // Draw the card
+  entry.state = CardState::IN_HAND;
+  this->draw_index++;
+  return true;
 }
 
 uint16_t DeckState::card_id_for_card_ref(uint16_t card_ref) const {
@@ -201,12 +205,17 @@ void DeckState::do_mulligan(bool is_nte) {
       this->card_refs[index + 5] = temp_ref;
     }
 
+    auto s = this->server.lock();
+    if (!s) {
+      throw runtime_error("server is missing");
+    }
+
     // Shuffle the deck, except the first 5 cards (which are about to be drawn).
     size_t max = this->num_drawable_cards() - 5;
     uint8_t base_index = this->draw_index + 5;
     for (size_t z = 0; z < this->card_refs.size(); z++) {
-      uint8_t index1 = this->random_crypt->next() % max;
-      uint8_t index2 = this->random_crypt->next() % max;
+      uint8_t index1 = s->get_random(max);
+      uint8_t index2 = s->get_random(max);
       uint16_t temp_ref = this->card_refs[base_index + index1];
       this->card_refs[base_index + index1] = this->card_refs[base_index + index2];
       this->card_refs[base_index + index2] = temp_ref;
@@ -258,6 +267,11 @@ void DeckState::set_card_discarded(uint16_t card_ref) {
 
 void DeckState::shuffle() {
   if (this->shuffle_enabled) {
+    auto s = this->server.lock();
+    if (!s) {
+      throw runtime_error("server is missing");
+    }
+
     size_t max = this->num_drawable_cards();
     for (size_t z = 0; z < this->card_refs.size(); z++) {
       // Note: This is the way Sega originally implemented shuffling - they just
@@ -265,8 +279,8 @@ void DeckState::shuffle() {
       // instead swap each item with another random item (possibly itself) that
       // doesn't appear earlier than it in the array, but this is not what Sega
       // did.
-      uint8_t index1 = this->draw_index + this->random_crypt->next() % max;
-      uint8_t index2 = this->draw_index + this->random_crypt->next() % max;
+      uint8_t index1 = this->draw_index + s->get_random(max);
+      uint8_t index2 = this->draw_index + s->get_random(max);
       uint16_t temp_ref = this->card_refs[index1];
       this->card_refs[index1] = this->card_refs[index2];
       this->card_refs[index2] = temp_ref;

src/Episode3/DeckState.hh

@@ -10,8 +10,10 @@
 
 namespace Episode3 {
 
+class Server;
+
 struct NameEntry {
-  /* 00 */ parray<char, 0x10> name;
+  /* 00 */ pstring<TextEncoding::MARKED, 0x10> name;
   /* 10 */ uint8_t client_id;
   /* 11 */ uint8_t present;
   /* 12 */ uint8_t is_cpu_player;
@@ -20,10 +22,10 @@ struct NameEntry {
 
   NameEntry();
   void clear();
-} __attribute__((packed));
+} __packed_ws__(NameEntry, 0x14);
 
 struct DeckEntry {
-  /* 00 */ pstring<TextEncoding::SJIS, 0x10> name;
+  /* 00 */ pstring<TextEncoding::MARKED, 0x10> name;
   /* 10 */ le_uint32_t team_id;
   /* 14 */ parray<le_uint16_t, 31> card_ids;
   // If the following flag is not set to 3, then the God Whim assist effect can
@@ -37,7 +39,7 @@ struct DeckEntry {
 
   DeckEntry();
   void clear();
-} __attribute__((packed));
+} __packed_ws__(DeckEntry, 0x58);
 
 uint8_t index_for_card_ref(uint16_t card_ref);
 uint8_t client_id_for_card_ref(uint16_t card_ref);
@@ -57,13 +59,13 @@ public:
   DeckState(
       uint8_t client_id,
       const parray<CardIDT, 0x1F>& card_ids,
-      std::shared_ptr<PSOLFGEncryption> random_crypt)
-      : client_id(client_id),
+      std::shared_ptr<Server> server)
+      : server(server),
+        client_id(client_id),
         draw_index(1),
         card_ref_base(this->client_id << 8),
         shuffle_enabled(true),
-        loop_enabled(true),
-        random_crypt(random_crypt) {
+        loop_enabled(true) {
     for (size_t z = 0; z < card_ids.size(); z++) {
       auto& e = this->entries[z];
       e.card_id = card_ids[z];
@@ -99,6 +101,8 @@ public:
   void print(FILE* stream, std::shared_ptr<const CardIndex> card_index = nullptr) const;
 
 private:
+  std::weak_ptr<Server> server;
+
   struct CardEntry {
     uint16_t card_id;
     uint8_t deck_index;
@@ -111,8 +115,6 @@ private:
   bool loop_enabled;
   parray<CardEntry, 31> entries;
   parray<uint16_t, 31> card_refs;
-
-  std::shared_ptr<PSOLFGEncryption> random_crypt;
 };
 
 } // namespace Episode3

src/Episode3/MapState.cc

@@ -99,17 +99,4 @@ MapAndRulesStateTrial::operator MapAndRulesState() const {
   return ret;
 }
 
-OverlayState::OverlayState() {
-  this->clear();
-}
-
-void OverlayState::clear() {
-  for (size_t y = 0; y < this->tiles.size(); y++) {
-    this->tiles[y].clear(0);
-  }
-  this->unused1.clear(0);
-  this->unused2.clear(0);
-  this->trap_card_ids_nte.clear(0);
-}
-
 } // namespace Episode3

src/Episode3/MapState.hh

@@ -20,7 +20,7 @@ struct MapState {
   void clear();
 
   void print(FILE* stream) const;
-} __attribute__((packed));
+} __packed_ws__(MapState, 0x110);
 
 struct MapAndRulesState {
   /* 0000 */ MapState map;
@@ -45,7 +45,7 @@ struct MapAndRulesState {
 
   void set_occupied_bit_for_tile(uint8_t x, uint8_t y);
   void clear_occupied_bit_for_tile(uint8_t x, uint8_t y);
-} __attribute__((packed));
+} __packed_ws__(MapAndRulesState, 0x138);
 
 struct MapAndRulesStateTrial {
   /* 0000 */ MapState map;
@@ -65,16 +65,6 @@ struct MapAndRulesStateTrial {
   MapAndRulesStateTrial() = default;
   MapAndRulesStateTrial(const MapAndRulesState& state);
   operator MapAndRulesState() const;
-} __attribute__((packed));
-
-struct OverlayState {
-  parray<parray<uint8_t, 0x10>, 0x10> tiles;
-  parray<le_uint32_t, 5> unused1;
-  parray<le_uint32_t, 0x10> unused2;
-  parray<le_uint16_t, 0x10> trap_card_ids_nte; // Unused on non-NTE
-
-  OverlayState();
-  void clear();
-} __attribute__((packed));
+} __packed_ws__(MapAndRulesStateTrial, 0x130);
 
 } // namespace Episode3

src/Episode3/PlayerState.cc

@@ -41,6 +41,7 @@ PlayerState::PlayerState(uint8_t client_id, shared_ptr<Server> server)
 
 void PlayerState::init() {
   auto s = this->server();
+  auto log = s->log_stack("PlayerState::init: ");
 
   if (s->player_states.at(this->client_id).get() != this) {
     // Note: The original code handles this, but we don't. This appears not to
@@ -48,7 +49,7 @@ void PlayerState::init() {
     throw logic_error("replacing a player state object is not permitted");
   }
 
-  this->deck_state = make_shared<DeckState>(this->client_id, s->deck_entries[client_id]->card_ids, s->options.random_crypt);
+  this->deck_state = make_shared<DeckState>(this->client_id, s->deck_entries[client_id]->card_ids, s);
   if (s->map_and_rules->rules.disable_deck_shuffle) {
     this->deck_state->disable_shuffle();
   }
@@ -131,6 +132,11 @@ shared_ptr<const Server> PlayerState::server() const {
   return s;
 }
 
+bool PlayerState::is_alive() const {
+  auto sc_card = this->get_sc_card();
+  return (sc_card && !(sc_card->card_flags & 2));
+}
+
 bool PlayerState::draw_cards_allowed() const {
   if (this->assist_flags & AssistFlag::IS_SKIPPING_TURN) {
     return false;
@@ -223,8 +229,7 @@ void PlayerState::apply_assist_card_effect_on_set(
 
       size_t log_index;
       for (log_index = 0; log_index < 0x10; log_index++) {
-        auto ce = s->definition_for_card_ref(
-            this->discard_log_card_refs[log_index]);
+        auto ce = s->definition_for_card_ref(this->discard_log_card_refs[log_index]);
         if (ce && ((ce->def.type == CardType::ITEM || ce->def.type == CardType::CREATURE))) {
           break;
         }
@@ -608,7 +613,7 @@ void PlayerState::discard_and_redraw_hand() {
   }
 
   if (!s->options.is_nte()) {
-    G_Unknown_Ep3_6xB4x2C cmd;
+    G_EnqueueAnimation_Ep3_6xB4x2C cmd;
     cmd.change_type = 3;
     cmd.client_id = this->client_id;
     cmd.card_refs.clear(0xFFFF);
@@ -712,7 +717,7 @@ bool PlayerState::do_mulligan() {
   }
 
   if (!s->options.is_nte()) {
-    G_Unknown_Ep3_6xB4x2C cmd;
+    G_EnqueueAnimation_Ep3_6xB4x2C cmd;
     cmd.change_type = 3;
     cmd.client_id = this->client_id;
     cmd.card_refs.clear(0xFFFF);
@@ -832,7 +837,7 @@ vector<uint16_t> PlayerState::get_all_cards_within_range(
     uint8_t target_team_id) const {
   auto s = this->server();
 
-  auto log = this->server()->log_stack("get_all_cards_within_range: ");
+  auto log = s->log_stack("get_all_cards_within_range: ");
   string loc_str = loc.str();
   log.debug("loc=%s, target_team_id=%02hhX", loc_str.c_str(), target_team_id);
 
@@ -1003,7 +1008,7 @@ bool PlayerState::move_card_to_location_by_card_index(size_t card_index, const L
   this->send_6xB4x04_if_needed();
   s->send_6xB4x05();
   s->send_6xB4x39();
-  s->card_special->unknown_80244AA8(card);
+  s->card_special->apply_effects_after_card_move(card);
   return true;
 }
 
@@ -1415,7 +1420,7 @@ bool PlayerState::set_card_from_hand(
   s->send_6xB4x05();
 
   if (!is_nte) {
-    G_Unknown_Ep3_6xB4x4A cmd;
+    G_AddToSetCardLog_Ep3_6xB4x4A cmd;
     cmd.card_refs.clear(0xFFFF);
     cmd.card_refs[0] = card_ref;
     cmd.client_id = this->client_id;
@@ -1754,30 +1759,45 @@ int16_t PlayerState::get_assist_turns_remaining() {
 
 bool PlayerState::set_action_cards_for_action_state(const ActionState& pa) {
   auto s = this->server();
+  auto log = s->log_stack("set_action_cards_for_action_state: ");
   bool is_nte = s->options.is_nte();
 
   auto attacker_card = s->card_for_set_card_ref(pa.attacker_card_ref);
   if (attacker_card) {
+    log.debug("attacker card present");
     attacker_card->card_flags |= 0x100;
   }
 
   auto action_type = s->ruler_server->get_pending_action_type(pa);
+  if (action_type == ActionType::DEFENSE) {
+    log.debug("action type is DEFENSE");
+  } else if (action_type == ActionType::ATTACK) {
+    log.debug("action type is ATTACK");
+  } else {
+    log.debug("action type is UNKNOWN");
+  }
   if (!is_nte) {
-    this->subtract_or_check_atk_or_def_points_for_action(pa, 1);
+    log.debug("(non-nte) subtracting action points");
+    this->subtract_or_check_atk_or_def_points_for_action(pa, true);
   }
 
   if (action_type == ActionType::ATTACK) {
     auto card = s->card_for_set_card_ref(pa.attacker_card_ref);
     if (card) {
       card->loc.direction = pa.facing_direction;
+      log.debug("set facing direction to %s", phosg::name_for_enum(card->loc.direction));
 
-      G_Unknown_Ep3_6xB4x4A cmd;
+      G_AddToSetCardLog_Ep3_6xB4x4A cmd;
       cmd.card_refs.clear(0xFFFF);
       cmd.client_id = this->client_id;
       cmd.round_num = s->get_round_num();
       cmd.entry_count = 0;
       size_t z = 0;
       do {
+        if (log.should_log(phosg::LogLevel::DEBUG)) {
+          string ref_str = s->debug_str_for_card_ref(pa.action_card_refs[z]);
+          log.debug("on action card ref %s", ref_str.c_str());
+        }
         card->unknown_80237A90(pa, pa.action_card_refs[z]);
         card->unknown_802379BC(pa.action_card_refs[z]);
         if (!is_nte) {
@@ -1811,6 +1831,10 @@ bool PlayerState::set_action_cards_for_action_state(const ActionState& pa) {
     for (size_t z = 0; (z < 4 * 9) && (pa.target_card_refs[z] != 0xFFFF); z++) {
       auto target_card = s->card_for_set_card_ref(pa.target_card_refs[z]);
       if (target_card) {
+        if (log.should_log(phosg::LogLevel::DEBUG)) {
+          string ref_str = s->debug_str_for_card_ref(pa.target_card_refs[z]);
+          log.debug("on target card ref %s", ref_str.c_str());
+        }
         target_card->unknown_802379DC(pa);
         if (!is_nte) {
           if (this->client_id == target_card->get_client_id()) {
@@ -1823,7 +1847,7 @@ bool PlayerState::set_action_cards_for_action_state(const ActionState& pa) {
       }
     }
     if (!is_nte) {
-      G_Unknown_Ep3_6xB4x4A cmd;
+      G_AddToSetCardLog_Ep3_6xB4x4A cmd;
       cmd.card_refs.clear(0xFFFF);
       cmd.client_id = this->client_id;
       cmd.round_num = s->get_round_num();
@@ -1833,9 +1857,14 @@ bool PlayerState::set_action_cards_for_action_state(const ActionState& pa) {
     }
   }
   if (is_nte) {
+    log.debug("(nte) subtracting action points");
     this->subtract_or_check_atk_or_def_points_for_action(pa, 1);
   }
   for (size_t z = 0; (z < pa.action_card_refs.size()) && (pa.action_card_refs[z] != 0xFFFF); z++) {
+    if (log.should_log(phosg::LogLevel::DEBUG)) {
+      string ref_str = s->debug_str_for_card_ref(pa.action_card_refs[z]);
+      log.debug("discarding %s from hand", ref_str.c_str());
+    }
     this->discard_ref_from_hand(pa.action_card_refs[z]);
   }
   this->update_hand_and_equip_state_and_send_6xB4x02_if_needed();
@@ -1960,44 +1989,23 @@ void PlayerState::roll_main_dice_or_apply_after_effects() {
   // the non-NTE logic and assign the dice ranges at battle start time to yield
   // the NTE behavior. (See RulesTrial in DataIndexes.cc for how this is done.)
 
-  uint8_t min_atk_dice = rules.min_dice;
-  uint8_t max_atk_dice = rules.max_dice;
-  if (min_atk_dice == 0) {
-    min_atk_dice = 1;
-  }
-  if (max_atk_dice == 0) {
-    max_atk_dice = 6;
-  }
-  if (max_atk_dice < min_atk_dice) {
-    uint8_t t = max_atk_dice;
-    max_atk_dice = min_atk_dice;
-    min_atk_dice = t;
-  }
-  uint8_t atk_dice_range_width = (max_atk_dice - min_atk_dice) + 1;
+  bool is_1p_2v1 = (s->team_client_count.at(this->get_team_id()) < s->team_client_count[this->get_team_id() ^ 1]);
+
+  auto atk_range = rules.atk_dice_range(is_1p_2v1);
+  auto def_range = rules.def_dice_range(is_1p_2v1);
+
+  uint8_t atk_dice_range_width = (atk_range.second - atk_range.first) + 1;
   if (atk_dice_range_width < 2) {
-    this->dice_results[0] = min_atk_dice;
+    this->dice_results[0] = atk_range.first;
   } else {
-    this->dice_results[0] = min_atk_dice + s->get_random(atk_dice_range_width);
+    this->dice_results[0] = atk_range.first + s->get_random(atk_dice_range_width);
   }
 
-  uint8_t min_def_dice = rules.min_def_dice() ? rules.min_def_dice() : rules.min_dice;
-  uint8_t max_def_dice = rules.max_def_dice() ? rules.max_def_dice() : rules.max_dice;
-  if (min_def_dice == 0) {
-    min_def_dice = 1;
-  }
-  if (max_def_dice == 0) {
-    max_def_dice = 6;
-  }
-  if (max_def_dice < min_def_dice) {
-    uint8_t t = max_def_dice;
-    max_def_dice = min_def_dice;
-    min_def_dice = t;
-  }
-  uint8_t def_dice_range_width = (max_def_dice - min_def_dice) + 1;
+  uint8_t def_dice_range_width = (def_range.second - def_range.first) + 1;
   if (def_dice_range_width < 2) {
-    this->dice_results[1] = min_def_dice;
+    this->dice_results[1] = def_range.first;
   } else {
-    this->dice_results[1] = min_def_dice + s->get_random(def_dice_range_width);
+    this->dice_results[1] = def_range.first + s->get_random(def_dice_range_width);
   }
 
   bool should_exchange = false;

src/Episode3/PlayerState.hh

@@ -47,6 +47,8 @@ public:
   std::shared_ptr<Server> server();
   std::shared_ptr<const Server> server() const;
 
+  bool is_alive() const;
+
   bool draw_cards_allowed() const;
   void apply_assist_card_effect_on_set(std::shared_ptr<PlayerState> setter_ps);
   void apply_dice_effects();
@@ -148,7 +150,7 @@ private:
 
 public:
   std::shared_ptr<Card> sc_card;
-  std::shared_ptr<Card> set_cards[8];
+  bcarray<std::shared_ptr<Card>, 8> set_cards;
   uint8_t client_id;
   uint16_t num_mulligans_allowed;
   CardType sc_card_type;

src/Episode3/PlayerStateSubordinates.cc

@@ -6,22 +6,6 @@ using namespace std;
 
 namespace Episode3 {
 
-template <size_t Count>
-std::string string_for_refs(const parray<le_uint16_t, Count>& card_refs) {
-  string ret = "[";
-  for (size_t z = 0; z < Count; z++) {
-    if (card_refs[z] != 0xFFFF) {
-      ret += string_printf("%zu:@$%04X ", z, card_refs[z].load());
-    }
-  }
-  if (!ret.empty()) {
-    ret.back() = ']'; // Replace the ' ' from the last added item
-  } else {
-    ret.push_back(']');
-  }
-  return ret;
-}
-
 Condition::Condition() {
   this->clear();
 }
@@ -77,20 +61,22 @@ void Condition::clear_FF() {
   this->unknown_a8 = 0xFF;
 }
 
-std::string Condition::str() const {
-  return string_printf(
+std::string Condition::str(shared_ptr<const Server> s) const {
+  auto card_ref_str = s->debug_str_for_card_ref(this->card_ref);
+  auto giver_ref_str = s->debug_str_for_card_ref(this->condition_giver_card_ref);
+  return phosg::string_printf(
       "Condition[type=%s, turns=%hhu, a_arg=%hhd, dice=%hhu, flags=%02hhX, "
-      "def_eff_index=%hhu, ref=@%04hX, value=%hd, giver_ref=@%04hX "
+      "def_eff_index=%hhu, ref=%s, value=%hd, giver_ref=%s "
       "percent=%hhu value8=%hd order=%hu a8=%hu]",
-      name_for_condition_type(this->type),
+      phosg::name_for_enum(this->type),
       this->remaining_turns,
       this->a_arg_value,
       this->dice_roll_value,
       this->flags,
       this->card_definition_effect_index,
-      this->card_ref.load(),
+      card_ref_str.c_str(),
       this->value.load(),
-      this->condition_giver_card_ref.load(),
+      giver_ref_str.c_str(),
       this->random_percent,
       this->value8,
       this->order,
@@ -114,13 +100,15 @@ void EffectResult::clear() {
   this->dice_roll_value = 0;
 }
 
-std::string EffectResult::str() const {
-  return string_printf(
-      "EffectResult[att_ref=@%04hX, target_ref=@%04hX, value=%hhd, "
+std::string EffectResult::str(shared_ptr<const Server> s) const {
+  string attacker_ref_str = s->debug_str_for_card_ref(this->attacker_card_ref);
+  string target_ref_str = s->debug_str_for_card_ref(this->target_card_ref);
+  return phosg::string_printf(
+      "EffectResult[att_ref=%s, target_ref=%s, value=%hhd, "
       "cur_hp=%hhd, ap=%hhd, tp=%hhd, flags=%02hhX, op=%hhd, "
       "cond_index=%hhu, dice=%hhu]",
-      this->attacker_card_ref.load(),
-      this->target_card_ref.load(),
+      attacker_ref_str.c_str(),
+      target_ref_str.c_str(),
       this->value,
       this->current_hp,
       this->ap,
@@ -148,12 +136,13 @@ bool CardShortStatus::operator!=(const CardShortStatus& other) const {
   return !this->operator==(other);
 }
 
-std::string CardShortStatus::str() const {
+std::string CardShortStatus::str(shared_ptr<const Server> s) const {
   string loc_s = this->loc.str();
-  return string_printf(
-      "CardShortStatus[ref=@%04hX, cur_hp=%hd, flags=%08" PRIX32 ", loc=%s, "
+  string ref_str = s->debug_str_for_card_ref(this->card_ref);
+  return phosg::string_printf(
+      "CardShortStatus[ref=%s, cur_hp=%hd, flags=%08" PRIX32 ", loc=%s, "
       "u1=%04hX, max_hp=%hhd, u2=%hhu]",
-      this->card_ref.load(),
+      ref_str.c_str(),
       this->current_hp.load(),
       this->card_flags.load(),
       loc_s.c_str(),
@@ -195,23 +184,27 @@ void ActionState::clear() {
   this->original_attacker_card_ref = 0xFFFF;
   this->target_card_refs.clear(0xFFFF);
   this->action_card_refs.clear(0xFFFF);
+  this->unused2 = 0xFFFF;
 }
 
-std::string ActionState::str() const {
-  string target_refs_s = string_for_refs(this->target_card_refs);
-  string action_refs_s = string_for_refs(this->action_card_refs);
-  return string_printf(
-      "ActionState[client=%hu, u=%hhu, facing=%s, attacker_ref=@%04hX, "
-      "def_ref=@%04hX, target_refs=%s, action_refs=%s, "
-      "orig_attacker_ref=@%04hX]",
+std::string ActionState::str(shared_ptr<const Server> s) const {
+  string attacker_ref_s = s->debug_str_for_card_ref(this->attacker_card_ref);
+  string defense_ref_s = s->debug_str_for_card_ref(this->defense_card_ref);
+  string original_attacker_ref_s = s->debug_str_for_card_ref(this->original_attacker_card_ref);
+  string target_refs_s = s->debug_str_for_card_refs(this->target_card_refs);
+  string action_refs_s = s->debug_str_for_card_refs(this->action_card_refs);
+  return phosg::string_printf(
+      "ActionState[client=%hX, u=%hhu, facing=%s, attacker_ref=%s, "
+      "def_ref=%s, target_refs=%s, action_refs=%s, "
+      "orig_attacker_ref=%s]",
       this->client_id.load(),
       this->unused,
-      name_for_direction(this->facing_direction),
-      this->attacker_card_ref.load(),
-      this->defense_card_ref.load(),
+      phosg::name_for_enum(this->facing_direction),
+      attacker_ref_s.c_str(),
+      defense_ref_s.c_str(),
       target_refs_s.c_str(),
       action_refs_s.c_str(),
-      this->original_attacker_card_ref.load());
+      original_attacker_ref_s.c_str());
 }
 
 ActionChain::ActionChain() {
@@ -245,28 +238,29 @@ bool ActionChain::operator!=(const ActionChain& other) const {
   return !this->operator==(other);
 }
 
-std::string ActionChain::str() const {
-  string attack_action_card_refs_s = string_for_refs(this->attack_action_card_refs);
-  string target_card_refs_s = string_for_refs(this->target_card_refs);
-  return string_printf(
+std::string ActionChain::str(shared_ptr<const Server> s) const {
+  string acting_card_ref_s = s->debug_str_for_card_ref(this->acting_card_ref);
+  string unknown_card_ref_a3_s = s->debug_str_for_card_ref(this->unknown_card_ref_a3);
+  string attack_action_card_refs_s = s->debug_str_for_card_refs(this->attack_action_card_refs);
+  string target_card_refs_s = s->debug_str_for_card_refs(this->target_card_refs);
+  return phosg::string_printf(
       "ActionChain[eff_ap=%hhd, eff_tp=%hhd, ap_bonus=%hhd, damage=%hhd, "
-      "acting_ref=@%04hX, unknown_ref_a3=@%04hX, "
-      "attack_action_refs=%s, attack_action_ref_count=%hhu, "
-      "medium=%s, target_ref_count=%hhu, subphase=%s, "
-      "strikes=%hhu, damage_mult=%hhd, attack_num=%hhu, "
+      "acting_ref=%s, unknown_ref_a3=%s, attack_action_refs=%s, "
+      "attack_action_ref_count=%hhu, medium=%s, target_ref_count=%hhu, "
+      "subphase=%s, strikes=%hhu, damage_mult=%hhd, attack_num=%hhu, "
       "tp_bonus=%hhd, phys_bonus_nte=%hhu, tech_bonus_nte=%hhu, card_ap=%hhd, "
       "card_tp=%hhd, flags=%08" PRIX32 ", target_refs=%s]",
       this->effective_ap,
       this->effective_tp,
       this->ap_effect_bonus,
       this->damage,
-      this->acting_card_ref.load(),
-      this->unknown_card_ref_a3.load(),
+      acting_card_ref_s.c_str(),
+      unknown_card_ref_a3_s.c_str(),
       attack_action_card_refs_s.c_str(),
       this->attack_action_card_ref_count,
-      name_for_attack_medium(this->attack_medium),
+      phosg::name_for_enum(this->attack_medium),
       this->target_card_ref_count,
-      name_for_action_subphase(this->action_subphase),
+      phosg::name_for_enum(this->action_subphase),
       this->strike_count,
       this->damage_multiplier,
       this->attack_number,
@@ -338,17 +332,17 @@ bool ActionChainWithConds::operator!=(const ActionChainWithConds& other) const {
   return !this->operator==(other);
 }
 
-std::string ActionChainWithConds::str() const {
+std::string ActionChainWithConds::str(shared_ptr<const Server> s) const {
   string ret = "ActionChainWithConds[chain=";
-  ret += this->chain.str();
+  ret += this->chain.str(s);
   ret += ", conds=[";
   for (size_t z = 0; z < this->conditions.size(); z++) {
     if (this->conditions[z].type != ConditionType::NONE) {
-      if (ret.back() != '=') {
+      if (ret.back() != '[') {
         ret += ", ";
       }
-      ret += string_printf("%zu:", z);
-      ret += this->conditions[z].str();
+      ret += phosg::string_printf("%zu:", z);
+      ret += this->conditions[z].str(s);
     }
   }
   ret += "]]";
@@ -581,19 +575,20 @@ bool ActionMetadata::operator!=(const ActionMetadata& other) const {
   return !this->operator==(other);
 }
 
-std::string ActionMetadata::str() const {
-  string target_card_refs_s = string_for_refs(this->target_card_refs);
-  string defense_card_refs_s = string_for_refs(this->defense_card_refs);
-  string original_attacker_card_refs_s = string_for_refs(this->original_attacker_card_refs);
-  return string_printf(
-      "ActionMetadata[ref=@%04hX, target_ref_count=%hhu, def_ref_count=%hhu, "
+std::string ActionMetadata::str(shared_ptr<const Server> s) const {
+  string card_ref_s = s->debug_str_for_card_ref(this->card_ref);
+  string target_card_refs_s = s->debug_str_for_card_refs(this->target_card_refs);
+  string defense_card_refs_s = s->debug_str_for_card_refs(this->defense_card_refs);
+  string original_attacker_card_refs_s = s->debug_str_for_card_refs(this->original_attacker_card_refs);
+  return phosg::string_printf(
+      "ActionMetadata[ref=%s, target_ref_count=%hhu, def_ref_count=%hhu, "
       "subphase=%s, def_power=%hhd, def_bonus=%hhd, "
       "att_bonus=%hhd, flags=%08" PRIX32 ", target_refs=%s, "
       "defense_refs=%s, original_attacker_refs=%s]",
-      this->card_ref.load(),
+      card_ref_s.c_str(),
       this->target_card_ref_count,
       this->defense_card_ref_count,
-      name_for_action_subphase(this->action_subphase),
+      phosg::name_for_enum(this->action_subphase),
       this->defense_power,
       this->defense_bonus,
       this->attack_bonus,
@@ -679,20 +674,22 @@ HandAndEquipState::HandAndEquipState() {
   this->clear();
 }
 
-std::string HandAndEquipState::str() const {
-  string hand_card_refs_s = string_for_refs(this->hand_card_refs);
-  string set_card_refs_s = string_for_refs(this->set_card_refs);
-  string hand_card_refs2_s = string_for_refs(this->hand_card_refs2);
-  string set_card_refs2_s = string_for_refs(this->set_card_refs2);
-  return string_printf(
+std::string HandAndEquipState::str(shared_ptr<const Server> s) const {
+  string assist_card_ref_s = s->debug_str_for_card_ref(this->assist_card_ref);
+  string assist_card_ref2_s = s->debug_str_for_card_ref(this->assist_card_ref2);
+  string assist_card_id_s = s->debug_str_for_card_id(this->assist_card_id);
+  string sc_card_ref_s = s->debug_str_for_card_ref(this->sc_card_ref);
+  string hand_card_refs_s = s->debug_str_for_card_refs(this->hand_card_refs);
+  string set_card_refs_s = s->debug_str_for_card_refs(this->set_card_refs);
+  string hand_card_refs2_s = s->debug_str_for_card_refs(this->hand_card_refs2);
+  string set_card_refs2_s = s->debug_str_for_card_refs(this->set_card_refs2);
+  return phosg::string_printf(
       "HandAndEquipState[dice=[%hhu, %hhu], atk=%hhu, def=%hhu, atk2=%hhu, "
-      "a1=%hhu, total_set_cost=%hhu, is_cpu=%hhu, "
-      "assist_flags=%08" PRIX32 ", hand_refs=%s, "
-      "assist_ref=@%04hX, set_refs=%s, sc_ref=@%04hX, "
-      "hand_refs2=%s, set_refs2=%s, assist_ref2=@%04hX, "
-      "assist_set_num=%hu, assist_card_id=#%04hX, "
-      "assist_turns=%hhu, assit_dely=%hhu, atk_bonus=%hhu, "
-      "def_bonus=%hhu, u2=[%hhu, %hhu]]",
+      "a1=%hhu, total_set_cost=%hhu, is_cpu=%hhu, assist_flags=%08" PRIX32 ", "
+      "hand_refs=%s, assist_ref=%s, set_refs=%s, sc_ref=%s, hand_refs2=%s, "
+      "set_refs2=%s, assist_ref2=%s, assist_set_num=%hu, assist_card_id=%s, "
+      "assist_turns=%hhu, assist_delay=%hhu, atk_bonus=%hhu, def_bonus=%hhu, "
+      "u2=[%hhu, %hhu]]",
       this->dice_results[0],
       this->dice_results[1],
       this->atk_points,
@@ -703,14 +700,14 @@ std::string HandAndEquipState::str() const {
       this->is_cpu_player,
       this->assist_flags.load(),
       hand_card_refs_s.c_str(),
-      this->assist_card_ref.load(),
+      assist_card_ref_s.c_str(),
       set_card_refs_s.c_str(),
-      this->sc_card_ref.load(),
+      sc_card_ref_s.c_str(),
       hand_card_refs2_s.c_str(),
       set_card_refs2_s.c_str(),
-      this->assist_card_ref2.load(),
+      assist_card_ref2_s.c_str(),
       this->assist_card_set_number.load(),
-      this->assist_card_id.load(),
+      assist_card_id_s.c_str(),
       this->assist_remaining_turns,
       this->assist_delay_turns,
       this->atk_bonuses,
@@ -861,7 +858,7 @@ static bool is_card_within_range(
     const parray<uint8_t, 9 * 9>& range,
     const Location& anchor_loc,
     const CardShortStatus& ss,
-    PrefixedLogger* log) {
+    phosg::PrefixedLogger* log) {
   if (ss.card_ref == 0xFFFF) {
     if (log) {
       log->debug("is_card_within_range: (false) ss.card_ref missing");
@@ -902,7 +899,7 @@ vector<uint16_t> get_card_refs_within_range(
     const parray<uint8_t, 9 * 9>& range,
     const Location& loc,
     const parray<CardShortStatus, 0x10>& short_statuses,
-    PrefixedLogger* log) {
+    phosg::PrefixedLogger* log) {
   vector<uint16_t> ret;
   if (is_card_within_range(range, loc, short_statuses[0], log)) {
     if (log) {

src/Episode3/PlayerStateSubordinates.hh

@@ -35,8 +35,8 @@ struct Condition {
   void clear();
   void clear_FF();
 
-  std::string str() const;
-} __attribute__((packed));
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(Condition, 0x10);
 
 struct EffectResult {
   /* 00 */ le_uint16_t attacker_card_ref;
@@ -55,10 +55,10 @@ struct EffectResult {
   bool operator==(const EffectResult& other) const;
   bool operator!=(const EffectResult& other) const;
 
-  std::string str() const;
-
   void clear();
-} __attribute__((packed));
+
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(EffectResult, 0x0C);
 
 struct CardShortStatus {
   /* 00 */ le_uint16_t card_ref;
@@ -77,8 +77,8 @@ struct CardShortStatus {
   void clear();
   void clear_FF();
 
-  std::string str() const;
-} __attribute__((packed));
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(CardShortStatus, 0x10);
 
 struct ActionState {
   /* 00 */ le_uint16_t client_id;
@@ -87,7 +87,8 @@ struct ActionState {
   /* 04 */ le_uint16_t attacker_card_ref;
   /* 06 */ le_uint16_t defense_card_ref;
   /* 08 */ parray<le_uint16_t, 4 * 9> target_card_refs;
-  /* 50 */ parray<le_uint16_t, 9> action_card_refs;
+  /* 50 */ parray<le_uint16_t, 8> action_card_refs;
+  /* 60 */ le_uint16_t unused2;
   /* 62 */ le_uint16_t original_attacker_card_ref;
   /* 64 */
 
@@ -97,8 +98,8 @@ struct ActionState {
 
   void clear();
 
-  std::string str() const;
-} __attribute__((packed));
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(ActionState, 0x64);
 
 struct ActionChain {
   // Note: Episode 3 Trial Edition has a different format for this structure.
@@ -133,8 +134,8 @@ struct ActionChain {
   void clear();
   void clear_FF();
 
-  std::string str() const;
-} __attribute__((packed));
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(ActionChain, 0x70);
 
 struct ActionChainWithConds {
   /* 0000 */ ActionChain chain;
@@ -171,8 +172,8 @@ struct ActionChainWithConds {
 
   uint8_t get_adjusted_move_ability_nte(uint8_t ability) const;
 
-  std::string str() const;
-} __attribute__((packed));
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(ActionChainWithConds, 0x100);
 
 struct ActionChainWithCondsTrial {
   /* 0000 */ int8_t effective_ap;
@@ -204,7 +205,7 @@ struct ActionChainWithCondsTrial {
   ActionChainWithCondsTrial() = default;
   ActionChainWithCondsTrial(const ActionChainWithConds& src);
   operator ActionChainWithConds() const;
-} __attribute__((packed));
+} __packed_ws__(ActionChainWithCondsTrial, 0x100);
 
 struct ActionMetadata {
   /* 00 */ le_uint16_t card_ref;
@@ -224,8 +225,6 @@ struct ActionMetadata {
   bool operator==(const ActionMetadata& other) const;
   bool operator!=(const ActionMetadata& other) const;
 
-  std::string str() const;
-
   void clear();
   void clear_FF();
 
@@ -240,7 +239,9 @@ struct ActionMetadata {
       uint16_t defense_card_ref,
       std::shared_ptr<Card> card,
       uint16_t original_attacker_card_ref);
-} __attribute__((packed));
+
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(ActionMetadata, 0x74);
 
 struct HandAndEquipState {
   /* 00 */ parray<uint8_t, 2> dice_results;
@@ -274,8 +275,8 @@ struct HandAndEquipState {
   void clear();
   void clear_FF();
 
-  std::string str() const;
-} __attribute__((packed));
+  std::string str(std::shared_ptr<const Server> s) const;
+} __packed_ws__(HandAndEquipState, 0x54);
 
 struct PlayerBattleStats {
   /* 00 */ le_uint16_t damage_given;
@@ -309,7 +310,7 @@ struct PlayerBattleStats {
 
   static uint8_t rank_for_score(float score);
   static const char* name_for_rank(uint8_t rank);
-} __attribute__((packed));
+} __packed_ws__(PlayerBattleStats, 0x28);
 
 struct PlayerBattleStatsTrial {
   /* 00 */ le_uint32_t damage_given = 0;
@@ -322,12 +323,12 @@ struct PlayerBattleStatsTrial {
   PlayerBattleStatsTrial() = default;
   PlayerBattleStatsTrial(const PlayerBattleStats& data);
   operator PlayerBattleStats() const;
-} __attribute__((packed));
+} __packed_ws__(PlayerBattleStatsTrial, 0x14);
 
 std::vector<uint16_t> get_card_refs_within_range(
     const parray<uint8_t, 9 * 9>& range,
     const Location& loc,
     const parray<CardShortStatus, 0x10>& short_statuses,
-    PrefixedLogger* log = nullptr);
+    phosg::PrefixedLogger* log = nullptr);
 
 } // namespace Episode3

src/Episode3/RulerServer.cc

@@ -2,6 +2,7 @@
 
 #include <optional>
 
+#include "DataIndexes.hh"
 #include "Server.hh"
 
 using namespace std;
@@ -14,8 +15,8 @@ void compute_effective_range(
     uint16_t card_id,
     const Location& loc,
     shared_ptr<const MapAndRulesState> map_and_rules,
-    PrefixedLogger* log) {
-  if (log && log->should_log(LogLevel::DEBUG)) {
+    phosg::PrefixedLogger* log) {
+  if (log && log->should_log(phosg::LogLevel::DEBUG)) {
     string loc_str = loc.str();
     log->debug("compute_effective_range: card_id=#%04hX, loc=%s", card_id, loc_str.c_str());
     log->debug("compute_effective_range: map_and_rules->map:");
@@ -939,7 +940,8 @@ bool RulerServer::check_usability_or_condition_apply(
     bool is_item_usability_check,
     AttackMedium attack_medium) const {
   auto s = this->server();
-  auto log = s->log_stack(string_printf("check_usability_or_condition_apply(%02hhX, #%04hX, %02hhX, #%04hX, #%04hX, %02hhX, %s, %s): ", client_id1, card_id1, client_id2, card_id2, card_id3, def_effect_index, is_item_usability_check ? "true" : "false", name_for_attack_medium(attack_medium)));
+  bool is_nte = s->options.is_nte();
+  auto log = s->log_stack(phosg::string_printf("check_usability_or_condition_apply(%02hhX, #%04hX, %02hhX, #%04hX, #%04hX, %02hhX, %s, %s): ", client_id1, card_id1, client_id2, card_id2, card_id3, def_effect_index, is_item_usability_check ? "true" : "false", phosg::name_for_enum(attack_medium)));
 
   if (static_cast<uint8_t>(attack_medium) & 0x80) {
     attack_medium = AttackMedium::UNKNOWN;
@@ -952,7 +954,7 @@ bool RulerServer::check_usability_or_condition_apply(
     log.debug("ce1 missing");
     return false;
   }
-  if (!s->options.is_nte() && (ce1->def.type == CardType::ITEM) && this->card_id_is_boss_sc(card_id2)) {
+  if (!is_nte && (ce1->def.type == CardType::ITEM) && this->card_id_is_boss_sc(card_id2)) {
     log.debug("ce1 is item and card_id2 is boss sc");
     return false;
   }
@@ -967,7 +969,7 @@ bool RulerServer::check_usability_or_condition_apply(
     }
     criterion_code = ce1->def.effects[def_effect_index].apply_criterion;
   }
-  log.debug("criterion_code=%s", name_for_criterion_code(criterion_code));
+  log.debug("criterion_code=%s", phosg::name_for_enum(criterion_code));
 
   // For item usability checks, prevent criteria that depend on player
   // positioning/team setup
@@ -989,7 +991,7 @@ bool RulerServer::check_usability_or_condition_apply(
   // second should not be given, so we'd return true if the criterion passes. If
   // neither of these cases apply, we should return false as a failsafe even if
   // the criterion passes. NTE did not have such a check.
-  bool ret = s->options.is_nte() || (!(def_effect_index & 0x80) || (client_id1 == client_id2)) || (client_id2 == 0xFF);
+  bool ret = is_nte || (!(def_effect_index & 0x80) || (client_id1 == client_id2)) || (client_id2 == 0xFF);
   switch (criterion_code) {
     case CriterionCode::NONE:
       return ret;
@@ -1413,7 +1415,7 @@ uint16_t RulerServer::compute_attack_or_defense_costs(
         return 99;
       }
       total_cost += (ce->def.self_cost + cost_bias);
-      if (card_class_is_tech_like(ce->def.card_class(), s->options.is_nte())) {
+      if (card_class_is_tech_like(ce->def.card_class(), is_nte)) {
         total_cost += tech_cost_bias;
       }
       total_ally_cost += ce->def.ally_cost;
@@ -1441,8 +1443,7 @@ uint16_t RulerServer::compute_attack_or_defense_costs(
       assist_cost_bias++;
     } else if (is_nte && (assist_effect == AssistEffect::DEFLATION)) {
       assist_cost_bias--;
-    } else if ((assist_effect == AssistEffect::BATTLE_ROYALE) &&
-        (pa.action_card_refs[0] == 0xFFFF)) {
+    } else if ((assist_effect == AssistEffect::BATTLE_ROYALE) && (pa.action_card_refs[0] == 0xFFFF)) {
       total_cost = 0;
       final_cost = 0;
     }
@@ -1469,36 +1470,52 @@ bool RulerServer::compute_effective_range_and_target_mode_for_attack(
     uint16_t* out_effective_card_id,
     TargetMode* out_effective_target_mode,
     uint16_t* out_orig_card_ref) const {
+  auto s = this->server();
+  bool is_nte = s->options.is_nte();
+  auto log = s->log_stack("compute_effective_range_and_target_mode_for_attack: ");
+
   size_t z;
-  for (z = 0; (z < 9) && (pa.action_card_refs[z] != 0xFFFF); z++) {
+  for (z = 0; (z < 8) && (pa.action_card_refs[z] != 0xFFFF); z++) {
   }
-  if (z >= 9) {
+  if (z >= 8) {
+    log.debug("too many action card refs");
     return false;
   }
+  log.debug("%zu action card refs", z);
   uint16_t card_ref = (z == 0) ? pa.attacker_card_ref : pa.action_card_refs[z - 1];
+  log.debug("base card ref = @%04hX", card_ref);
 
   uint16_t card_id = this->card_id_for_card_ref(card_ref);
   if (card_id == 0xFFFF) {
+    log.debug("card ref is broken");
     return false;
   }
 
   auto ce = this->definition_for_card_id(card_id);
   uint8_t client_id = client_id_for_card_ref(pa.attacker_card_ref);
   if ((client_id == 0xFF) || !ce) {
+    log.debug("card ref is broken or definition is missing");
     return false;
   }
 
   if (out_orig_card_ref) {
+    log.debug("orig_card_ref = @%04hX", card_ref);
     *out_orig_card_ref = card_ref;
   }
 
   auto target_mode = ce->def.target_mode;
   if (this->card_ref_or_sc_has_fixed_range(pa.attacker_card_ref)) {
+    const char* target_mode_name = name_for_target_mode(target_mode);
+    log.debug("attacker card ref @%04hX has fixed range; target mode is %s (%hhu)",
+        pa.attacker_card_ref.load(), target_mode_name, static_cast<uint8_t>(target_mode));
     card_id = this->card_id_for_card_ref(pa.attacker_card_ref);
-    if (!this->server()->options.is_nte()) {
+    if (!is_nte) {
       auto sc_ce = this->definition_for_card_id(card_id);
       if (sc_ce && (static_cast<uint8_t>(target_mode) < 6)) {
         target_mode = sc_ce->def.target_mode;
+        const char* target_mode_name = name_for_target_mode(target_mode);
+        log.debug("sc_ce overrides target mode with %s (%hhu)",
+            target_mode_name, static_cast<uint8_t>(target_mode));
       }
     }
   }
@@ -1508,8 +1525,10 @@ bool RulerServer::compute_effective_range_and_target_mode_for_attack(
     auto assist_effect = this->assist_server->get_active_assist_by_index(z);
     if (assist_effect == AssistEffect::SIMPLE) {
       card_id = this->card_id_for_card_ref(pa.attacker_card_ref);
+      log.debug("SIMPLE assist overrides card id with #%04hX", card_id);
     } else if (assist_effect == AssistEffect::HEAVY_FOG) {
       card_id = 0xFFFE;
+      log.debug("HEAVY_FOG assist overrides card id with #%04hX", card_id);
     }
   }
 
@@ -1630,8 +1649,7 @@ bool RulerServer::defense_card_can_apply_to_attack(
   return true;
 }
 
-bool RulerServer::defense_card_matches_any_attack_card_top_color(
-    const ActionState& pa) const {
+bool RulerServer::defense_card_matches_any_attack_card_top_color(const ActionState& pa) const {
   auto ce = this->definition_for_card_ref(pa.action_card_refs[0]);
   if (!ce) {
     throw runtime_error("defense card definition is missing");
@@ -1847,7 +1865,7 @@ int32_t RulerServer::error_code_for_client_setting_card(
         return -0x7E;
       }
 
-      int32_t x_offset, y_offset;
+      int32_t x_offset = 0, y_offset = 0;
       this->offsets_for_direction(summon_area_loc, &x_offset, &y_offset);
       if (x_offset == 0) {
         if ((loc->x < 1) && (loc->x >= this->map_and_rules->map.width - 1)) {
@@ -2041,21 +2059,28 @@ shared_ptr<const CardIndex::CardEntry> RulerServer::definition_for_card_id(uint3
 
 uint32_t RulerServer::get_card_id_with_effective_range(
     uint16_t card_ref, uint16_t card_id_override, TargetMode* out_target_mode) const {
+  auto log = this->server()->log_stack(phosg::string_printf("get_card_id_with_effective_range(@%04hX, #%04hX): ", card_ref, card_id_override));
+
   uint16_t card_id = (card_id_override == 0xFFFF)
       ? this->card_id_for_card_ref(card_ref)
       : card_id_override;
+  log.debug("card_id=#%04hX", card_id);
 
   if (card_id != 0xFFFF) {
     auto ce = this->definition_for_card_id(card_id);
     uint8_t client_id = client_id_for_card_ref(card_ref);
     if ((client_id != 0xFF) && ce) {
       TargetMode effective_target_mode = ce->def.target_mode;
+      log.debug("ce valid for #%04hX with effective target mode %s", card_id, name_for_target_mode(effective_target_mode));
 
       if (this->card_ref_or_sc_has_fixed_range(card_ref)) {
         // Undo the override that may have been passed in
-        auto ce = this->definition_for_card_id(this->card_id_for_card_ref(card_ref));
-        if (ce && (static_cast<uint8_t>(effective_target_mode) < 6)) {
-          effective_target_mode = ce->def.target_mode;
+        log.debug("@%04hX has FIXED_RANGE", card_ref);
+        card_id = this->card_id_for_card_ref(card_ref);
+        auto orig_ce = this->definition_for_card_id(card_id);
+        if (orig_ce && (static_cast<uint8_t>(effective_target_mode) < 6)) {
+          log.debug("ce valid for #%04hX with effective target mode %s; overriding to %s", card_id, name_for_target_mode(effective_target_mode), name_for_target_mode(orig_ce->def.target_mode));
+          effective_target_mode = orig_ce->def.target_mode;
         }
       }
 
@@ -2064,14 +2089,17 @@ uint32_t RulerServer::get_card_id_with_effective_range(
         auto eff = this->assist_server->get_active_assist_by_index(z);
         if (eff == AssistEffect::SIMPLE) {
           card_id = this->card_id_for_card_ref(card_ref);
+          log.debug("SIMPLE assist effect is active; using #%04hX for range", card_id);
         } else if (eff == AssistEffect::HEAVY_FOG) {
           card_id = 0xFFFE;
+          log.debug("HEAVY_FOG assist effect is active; limiting range to one tile in front");
         }
       }
 
       if (out_target_mode) {
         *out_target_mode = effective_target_mode;
       }
+      log.debug("results: card_id=#%04hX, target_mode=%s", card_id, name_for_target_mode(effective_target_mode));
     }
   }
 
@@ -2274,7 +2302,7 @@ bool RulerServer::is_attack_valid(const ActionState& pa) {
 
   size_t conditional_card_count = 0;
   size_t z;
-  for (z = 0; z < 9; z++) {
+  for (z = 0; z < 8; z++) {
     uint16_t right_card_ref = pa.action_card_refs[z];
     if (right_card_ref == 0xFFFF) {
       break;

src/Episode3/RulerServer.hh

@@ -19,7 +19,7 @@ void compute_effective_range(
     uint16_t card_id,
     const Location& loc,
     std::shared_ptr<const MapAndRulesState> map_and_rules,
-    PrefixedLogger* log = nullptr);
+    phosg::PrefixedLogger* log = nullptr);
 
 bool card_linkage_is_valid(
     std::shared_ptr<const CardIndex::CardEntry> right_def,
@@ -198,11 +198,11 @@ private:
   std::weak_ptr<Server> w_server;
 
 public:
-  std::shared_ptr<HandAndEquipState> hand_and_equip_states[4];
-  std::shared_ptr<parray<CardShortStatus, 0x10>> short_statuses[4];
-  std::shared_ptr<DeckEntry> deck_entries[4];
-  std::shared_ptr<parray<ActionChainWithConds, 9>> set_card_action_chains[4];
-  std::shared_ptr<parray<ActionMetadata, 9>> set_card_action_metadatas[4];
+  bcarray<std::shared_ptr<HandAndEquipState>, 4> hand_and_equip_states;
+  bcarray<std::shared_ptr<parray<CardShortStatus, 0x10>>, 4> short_statuses;
+  bcarray<std::shared_ptr<DeckEntry>, 4> deck_entries;
+  bcarray<std::shared_ptr<parray<ActionChainWithConds, 9>>, 4> set_card_action_chains;
+  bcarray<std::shared_ptr<parray<ActionMetadata, 9>>, 4> set_card_action_metadatas;
   std::shared_ptr<MapAndRulesState> map_and_rules;
   std::shared_ptr<StateFlags> state_flags;
   std::shared_ptr<AssistServer> assist_server;

src/Episode3/Server.cc

@@ -4,6 +4,7 @@
 #include <phosg/Time.hh>
 
 #include "../Loggers.hh"
+#include "../Revision.hh"
 #include "../SendCommands.hh"
 
 using namespace std;
@@ -11,7 +12,8 @@ using namespace std;
 namespace Episode3 {
 
 // This is (obviously) not the original string. The original string is:
-// "[V1][FINAL2.0] 03/09/13 15:30 by K.Toya"
+// "03/05/29 18:00 by K.Toya" (NTE)
+// "[V1][FINAL2.0] 03/09/13 15:30 by K.Toya" (Final)
 static const char* VERSION_SIGNATURE =
     "newserv Ep3 based on [V1][FINAL2.0] 03/09/13 15:30 by K.Toya";
 static const char* VERSION_SIGNATURE_NTE =
@@ -29,11 +31,15 @@ void Server::PresenceEntry::clear() {
 
 Server::Server(shared_ptr<Lobby> lobby, Options&& options)
     : lobby(lobby),
+      battle_record(lobby ? lobby->battle_record : nullptr),
       has_lobby(lobby != nullptr),
       options(std::move(options)),
       last_chosen_map(this->options.tournament ? this->options.tournament->get_map() : nullptr),
       tournament_match_result_sent(false),
       override_environment_number(0xFF),
+      def_dice_value_range_override(0xFF),
+      atk_dice_value_range_2v1_override(0xFF),
+      def_dice_value_range_2v1_override(0xFF),
       battle_finished(false),
       battle_in_progress(false),
       round_num(1),
@@ -77,7 +83,7 @@ Server::Server(shared_ptr<Lobby> lobby, Options&& options)
 
 Server::~Server() noexcept(false) {
   if (this->logger_stack.size() != 1) {
-    throw logic_error(string_printf("incorrect logger stack size: expected 1, received %zu", this->logger_stack.size()));
+    throw logic_error(phosg::string_printf("incorrect logger stack size: expected 1, received %zu", this->logger_stack.size()));
   }
   delete this->logger_stack.back();
 }
@@ -120,7 +126,7 @@ Server::StackLogger::StackLogger(const Server* s, const std::string& prefix)
   s->logger_stack.push_back(this);
 }
 
-Server::StackLogger::StackLogger(const Server* s, const std::string& prefix, LogLevel min_level)
+Server::StackLogger::StackLogger(const Server* s, const std::string& prefix, phosg::LogLevel min_level)
     : PrefixedLogger(prefix, min_level),
       server(s) {
   s->logger_stack.push_back(this);
@@ -160,6 +166,32 @@ const Server::StackLogger& Server::log() const {
   return *this->logger_stack.back();
 }
 
+std::string Server::debug_str_for_card_ref(uint16_t card_ref) const {
+  if (card_ref == 0xFFFF) {
+    return "@FFFF";
+  }
+  auto ce = this->definition_for_card_ref(card_ref);
+  if (ce) {
+    string name = ce->def.en_name.decode();
+    return phosg::string_printf("@%04hX (#%04" PRIX32 " %s)", card_ref, ce->def.card_id.load(), name.c_str());
+  } else {
+    return phosg::string_printf("@%04hX (missing)", card_ref);
+  }
+}
+
+std::string Server::debug_str_for_card_id(uint16_t card_id) const {
+  if (card_id == 0xFFFF) {
+    return "#FFFF";
+  }
+  auto ce = this->definition_for_card_id(card_id);
+  if (ce) {
+    string name = ce->def.en_name.decode();
+    return phosg::string_printf("#%04hX (%s)", card_id, name.c_str());
+  } else {
+    return phosg::string_printf("#%04hX (missing)", card_id);
+  }
+}
+
 int8_t Server::get_winner_team_id() const {
   // Note: This function is not part of the original implementation.
 
@@ -209,7 +241,7 @@ void Server::send(const void* data, size_t size, uint8_t command, bool enable_ma
         !(this->options.behavior_flags & BehaviorFlag::DISABLE_MASKING) &&
         (size >= 8)) {
       masked_data.assign(reinterpret_cast<const char*>(data), size);
-      uint8_t mask_key = (random_object<uint32_t>() % 0xFF) + 1;
+      uint8_t mask_key = (phosg::random_object<uint32_t>() % 0xFF) + 1;
       set_mask_for_ep3_game_command(masked_data.data(), masked_data.size(), mask_key);
       data = masked_data.data();
       size = masked_data.size();
@@ -223,37 +255,28 @@ void Server::send(const void* data, size_t size, uint8_t command, bool enable_ma
     for (auto watcher_l : l->watcher_lobbies) {
       send_command_if_not_loading(watcher_l, command, 0x00, data, size);
     }
-    if (l->battle_record && l->battle_record->writable()) {
-      l->battle_record->add_command(BattleRecord::Event::Type::BATTLE_COMMAND, data, size);
+    if (this->battle_record && this->battle_record->writable()) {
+      this->battle_record->add_command(BattleRecord::Event::Type::BATTLE_COMMAND, data, size);
     }
 
-  } else if (this->log().info("Generated command")) {
-    print_data(stderr, data, size);
+  } else if ((this->options.behavior_flags & BehaviorFlag::LOG_COMMANDS_IF_LOBBY_MISSING) &&
+      this->log().info("Generated command")) {
+    phosg::print_data(stderr, data, size, 0, nullptr, phosg::PrintDataFlags::PRINT_ASCII | phosg::PrintDataFlags::DISABLE_COLOR | phosg::PrintDataFlags::OFFSET_16_BITS);
   }
 }
 
 void Server::send_6xB4x46() const {
   // Note: This function is not part of the original implementation; it was
   // factored out from its callsites in this file and the strings were changed.
-  if (this->options.is_nte()) {
-    G_ServerVersionStrings_Ep3NTE_6xB4x46 cmd;
-    cmd.version_signature.encode(VERSION_SIGNATURE_NTE, 1);
-    cmd.date_str1.encode(format_time(this->options.card_index->definitions_mtime() * 1000000), 1);
-    this->send(cmd);
-  } else {
-    G_ServerVersionStrings_Ep3_6xB4x46 cmd;
-    cmd.version_signature.encode(VERSION_SIGNATURE, 1);
-    cmd.date_str1.encode(format_time(this->options.card_index->definitions_mtime() * 1000000), 1);
-    string date_str2 = string_printf(
-        "Random:%08" PRIX32 "+%08" PRIX32,
-        this->options.random_crypt->seed(),
-        this->options.random_crypt->absolute_offset());
-    if (this->last_chosen_map) {
-      date_str2 += string_printf(" Map:%08" PRIX32, this->last_chosen_map->map_number);
-    }
-    cmd.date_str2.encode(date_str2, 1);
-    this->send(cmd);
-  }
+
+  // NTE doesn't have the date_str2 field, but we send it anyway to make
+  // debugging easier.
+  G_ServerVersionStrings_Ep3_6xB4x46 cmd;
+  cmd.version_signature.encode(this->options.is_nte() ? VERSION_SIGNATURE_NTE : VERSION_SIGNATURE, 1);
+  cmd.date_str1.encode(phosg::format_time(this->options.card_index->definitions_mtime() * 1000000), 1);
+  string build_date = phosg::format_time(BUILD_TIMESTAMP);
+  cmd.date_str2.encode(phosg::string_printf("newserv %s compiled at %s", GIT_REVISION_HASH, build_date.c_str()), 1);
+  this->send(cmd);
 }
 
 string Server::prepare_6xB6x41_map_definition(shared_ptr<const MapIndex::Map> map, uint8_t language, bool is_nte) {
@@ -261,7 +284,7 @@ string Server::prepare_6xB6x41_map_definition(shared_ptr<const MapIndex::Map> ma
 
   const auto& compressed = vm->compressed(is_nte);
 
-  StringWriter w;
+  phosg::StringWriter w;
   uint32_t subcommand_size = (compressed.size() + sizeof(G_MapData_Ep3_6xB6x41) + 3) & (~3);
   w.put<G_MapData_Ep3_6xB6x41>({{{{0xB6, 0, 0}, subcommand_size}, 0x41, {}}, vm->map->map_number.load(), compressed.size(), 0});
   w.write(compressed);
@@ -325,7 +348,7 @@ __attribute__((format(printf, 2, 3))) void Server::send_debug_message_printf(con
   if (l && (this->options.behavior_flags & Episode3::BehaviorFlag::ENABLE_STATUS_MESSAGES)) {
     va_list va;
     va_start(va, fmt);
-    std::string buf = string_vprintf(fmt, va);
+    std::string buf = phosg::string_vprintf(fmt, va);
     va_end(va);
     send_text_message(l, buf);
   }
@@ -336,7 +359,7 @@ __attribute__((format(printf, 2, 3))) void Server::send_info_message_printf(cons
   if (l) {
     va_list va;
     va_start(va, fmt);
-    std::string buf = string_vprintf(fmt, va);
+    std::string buf = phosg::string_vprintf(fmt, va);
     va_end(va);
     send_text_message(l, buf);
   }
@@ -353,8 +376,7 @@ void Server::send_debug_command_received_message(uint8_t subsubcommand, const ch
   this->send_debug_message_printf("$C5*/CAx%02hhX %s", subsubcommand, description);
 }
 
-void Server::send_debug_message_if_error_code_nonzero(
-    uint8_t client_id, int32_t error_code) const {
+void Server::send_debug_message_if_error_code_nonzero(uint8_t client_id, int32_t error_code) const {
   if (error_code < 0) {
     this->send_debug_message_printf("$C4%hhu/ERROR -0x%zX", client_id, static_cast<ssize_t>(-error_code));
   } else if (error_code > 0) {
@@ -370,8 +392,7 @@ void Server::add_team_exp(uint8_t team_id, int32_t exp) {
     }
   }
 
-  this->team_exp[team_id] = clamp<int16_t>(
-      this->team_exp[team_id] + exp, 0, this->team_client_count[team_id] * 96);
+  this->team_exp[team_id] = clamp<int16_t>(this->team_exp[team_id] + exp, 0, this->team_client_count[team_id] * 96);
 
   uint8_t dice_boost = this->team_exp[team_id] / (this->team_client_count[team_id] * 12);
   this->card_special->adjust_dice_boost_if_team_has_condition_52(team_id, &dice_boost, 0);
@@ -453,29 +474,7 @@ shared_ptr<Card> Server::card_for_set_card_ref(uint16_t card_ref) {
 }
 
 shared_ptr<const Card> Server::card_for_set_card_ref(uint16_t card_ref) const {
-  // TODO: It'd be nice to deduplicate this function with the non-const version.
-  if (card_ref == 0xFFFF) {
-    return nullptr;
-  }
-  uint8_t client_id = client_id_for_card_ref(card_ref);
-  if (client_id == 0xFF) {
-    return nullptr;
-  }
-  auto ps = this->player_states.at(client_id);
-  if (!ps) {
-    return nullptr;
-  }
-  auto card = ps->get_sc_card();
-  if (card && (card->get_card_ref() == card_ref)) {
-    return card;
-  }
-  for (size_t set_index = 0; set_index < 8; set_index++) {
-    card = ps->get_set_card(set_index);
-    if (card && (card->get_card_ref() == card_ref)) {
-      return card;
-    }
-  }
-  return nullptr;
+  return const_cast<Server*>(this)->card_for_set_card_ref(card_ref);
 }
 
 uint16_t Server::card_id_for_card_ref(uint16_t card_ref) const {
@@ -752,6 +751,9 @@ void Server::destroy_cards_with_zero_hp() {
 void Server::determine_first_team_turn() {
   this->team_client_count[0] = this->map_and_rules->num_team0_players;
   this->team_client_count[1] = this->map_and_rules->num_players - this->team_client_count[0];
+  if (this->team_client_count[0] == 0 || this->team_client_count[1] == 0) {
+    throw runtime_error("one or both teams have no players");
+  }
   this->first_team_turn = 0xFF;
   while (this->first_team_turn == 0xFF) {
     uint8_t results[2] = {0, 0};
@@ -837,7 +839,7 @@ void Server::draw_phase_after() {
       // facilities used are different.
       uint64_t limit_5mins = this->map_and_rules->rules.overall_time_limit;
       uint64_t end_usecs = this->battle_start_usecs + (limit_5mins * 300 * 1000 * 1000);
-      if (now() >= end_usecs) {
+      if (phosg::now() >= end_usecs) {
         this->overall_time_expired = true;
       }
     }
@@ -946,44 +948,63 @@ void Server::end_action_phase() {
 }
 
 bool Server::enqueue_attack_or_defense(uint8_t client_id, ActionState* pa) {
+  auto log = this->log_stack("enqueue_attack_or_defense: ");
+  if (log.should_log(phosg::LogLevel::DEBUG)) {
+    string s = pa->str(this->shared_from_this());
+    log.debug("input: %s", s.c_str());
+  }
+
   if (client_id >= 4) {
     this->ruler_server->error_code3 = -0x78;
+    log.debug("failed: invalid client ID");
     return false;
   }
 
   auto ps = this->player_states[client_id];
   if (!ps) {
     this->ruler_server->error_code3 = -0x72;
+    log.debug("failed: player not present");
     return false;
   }
 
   if (pa->action_card_refs[0] == 0xFFFF) {
     if (pa->defense_card_ref != 0xFFFF) {
       pa->action_card_refs[0] = pa->defense_card_ref;
+      log.debug("moved defense card ref to action card ref 0");
     }
   } else {
     pa->defense_card_ref = pa->action_card_refs[0];
+    log.debug("moved action card ref 0 to defense card ref");
   }
 
   if (!this->ruler_server->is_attack_or_defense_valid(*pa)) {
+    log.debug("failed: attack or defense not valid");
     return false;
   }
 
   int16_t ally_atk_result = this->send_6xB4x33_remove_ally_atk_if_needed(*pa);
   if (ally_atk_result == 1) {
+    log.debug("pending: need ally approval");
     return true;
   } else if (ally_atk_result == -1) {
+    log.debug("failed: ally declined");
     return false;
   }
 
   if (this->num_pending_attacks >= 0x20) {
     this->ruler_server->error_code3 = -0x71;
+    log.debug("failed: too many pending attacks");
     return false;
   }
 
   size_t attack_index = this->num_pending_attacks++;
   this->pending_attacks[attack_index] = *pa;
+  if (log.should_log(phosg::LogLevel::DEBUG)) {
+    string pa_str = this->pending_attacks[attack_index].str(this->shared_from_this());
+    log.debug("set pending attack %zu: %s", attack_index, pa_str.c_str());
+  }
   ps->set_action_cards_for_action_state(*pa);
+  log.debug("set action cards");
   auto card = this->card_for_set_card_ref(this->send_6xB4x06_if_card_ref_invalid(pa->attacker_card_ref, 1));
   if (card) {
     card->card_flags |= 0x400;
@@ -1030,16 +1051,30 @@ shared_ptr<const PlayerState> Server::get_player_state(uint8_t client_id) const
   return this->player_states[client_id];
 }
 
+uint32_t Server::get_random_raw() {
+  le_uint32_t ret;
+  if (this->options.opt_rand_stream) {
+    this->options.opt_rand_stream->readx(&ret, sizeof(ret));
+  } else {
+    ret = random_from_optional_crypt(this->options.opt_rand_crypt);
+  }
+
+  if (this->battle_record && this->battle_record->writable()) {
+    this->battle_record->add_random_data(&ret, sizeof(ret));
+  }
+  return ret;
+}
+
 uint32_t Server::get_random(uint32_t max) {
   // The original implementation was essentially:
-  // return (static_cast<double>(this->random_crypt->next() >> 16) / 65536.0) * max
-  // This is unnecessarily complicated, so we instead just do this:
-  return this->options.random_crypt->next() % max;
+  // return (static_cast<double>(this->random_source->next() >> 16) / 65536.0) * max
+  // This is unnecessarily complicated and imprecise, so we instead just do:
+  return this->get_random_raw() % max;
 }
 
 float Server::get_random_float_0_1() {
   // This lacks some precision, but matches the original implementation.
-  return (static_cast<double>(this->options.random_crypt->next() >> 16) / 65536.0);
+  return (static_cast<double>(this->get_random_raw() >> 16) / 65536.0);
 }
 
 uint32_t Server::get_round_num() const {
@@ -1122,7 +1157,7 @@ void Server::move_phase_after() {
                 (abs(sc_card->loc.x - trap_x) < 2) &&
                 (abs(sc_card->loc.y - trap_y) < 2) &&
                 ps->replace_assist_card_by_id(trap_card_id)) {
-              G_Unknown_Ep3_6xB4x2C cmd;
+              G_EnqueueAnimation_Ep3_6xB4x2C cmd;
               cmd.change_type = 0x01;
               cmd.client_id = client_id;
               cmd.card_refs.clear(0xFFFF);
@@ -1318,6 +1353,13 @@ void Server::set_battle_started() {
   this->send_6xB4x05();
 }
 
+bool Server::player_can_receive_dice_boost(uint8_t client_id) const {
+  auto ps = this->player_states[client_id];
+  bool is_1p_2v1 = (this->team_client_count.at(ps->get_team_id()) < this->team_client_count[ps->get_team_id() ^ 1]);
+  const auto& rules = this->map_and_rules->rules;
+  return (rules.atk_dice_range(is_1p_2v1).second >= 3) || (rules.def_dice_range(is_1p_2v1).second >= 3);
+}
+
 void Server::set_client_id_ready_to_advance_phase(uint8_t client_id, BattlePhase battle_phase) {
   if (client_id >= 4) {
     return;
@@ -1332,7 +1374,10 @@ void Server::set_client_id_ready_to_advance_phase(uint8_t client_id, BattlePhase
     ps->assist_flags |= AssistFlag::READY_TO_END_PHASE;
     ps->update_hand_and_equip_state_and_send_6xB4x02_if_needed();
     if (this->battle_phase == BattlePhase::DICE) {
-      if (is_nte || !(ps->assist_flags & AssistFlag::ELIGIBLE_FOR_DICE_BOOST) || this->map_and_rules->rules.disable_dice_boost) {
+      if (is_nte ||
+          !(ps->assist_flags & AssistFlag::ELIGIBLE_FOR_DICE_BOOST) ||
+          this->map_and_rules->rules.disable_dice_boost ||
+          !this->player_can_receive_dice_boost(client_id)) {
         ps->assist_flags &= (~AssistFlag::ELIGIBLE_FOR_DICE_BOOST);
         ps->roll_main_dice_or_apply_after_effects();
         if (!is_nte && (ps->get_atk_points() < 3) && (ps->get_def_points() < 3)) {
@@ -1394,12 +1439,12 @@ void Server::set_phase_after() {
     if (ps) {
       auto card = ps->get_sc_card();
       if (card) {
-        this->card_special->apply_action_conditions(0x06, nullptr, card, is_nte ? 0x1F : 0x04, nullptr);
+        this->card_special->apply_action_conditions(EffectWhen::AFTER_SET_PHASE, nullptr, card, is_nte ? 0x1F : 0x04, nullptr);
       }
       for (size_t set_index = 0; set_index < 8; set_index++) {
         auto card = ps->get_set_card(set_index);
         if (card) {
-          this->card_special->apply_action_conditions(0x06, nullptr, card, is_nte ? 0x1F : 0x04, nullptr);
+          this->card_special->apply_action_conditions(EffectWhen::AFTER_SET_PHASE, nullptr, card, is_nte ? 0x1F : 0x04, nullptr);
         }
       }
     }
@@ -1486,8 +1531,8 @@ void Server::setup_and_start_battle() {
 
   this->setup_phase = SetupPhase::STARTER_ROLLS;
 
-  // Note: This is where original implementation re-seeds random_crypt (it uses
-  // time() as the seed value).
+  // Note: This is where original implementation re-seeds its random generator
+  // (it uses time() as the seed value).
 
   for (size_t z = 0; z < 4; z++) {
     if (!this->check_presence_entry(z)) {
@@ -1643,7 +1688,7 @@ void Server::setup_and_start_battle() {
     cmd.start_battle = 1;
     this->send(cmd);
   }
-  this->battle_start_usecs = now();
+  this->battle_start_usecs = phosg::now();
 
   this->send_6xB4x46();
 
@@ -1761,8 +1806,10 @@ const unordered_map<uint8_t, Server::handler_t> Server::subcommand_handlers({
 
 void Server::on_server_data_input(shared_ptr<Client> sender_c, const string& data) {
   auto header = check_size_t<G_CardBattleCommandHeader>(data, 0xFFFF);
-  if (header.size * 4 < data.size()) {
-    throw runtime_error("command is incomplete");
+  size_t expected_size = header.size * 4;
+  if (expected_size < data.size()) {
+    phosg::print_data(stderr, data);
+    throw runtime_error(phosg::string_printf("command is incomplete: expected %zX bytes, received %zX bytes", expected_size, data.size()));
   }
   if (header.subcommand != 0xB3) {
     throw runtime_error("server data command is not 6xB3");
@@ -1775,7 +1822,11 @@ void Server::on_server_data_input(shared_ptr<Client> sender_c, const string& dat
     throw runtime_error("unknown CAx subsubcommand");
   }
 
-  if (this->options.is_nte() || !header.mask_key) {
+  if (this->battle_record && this->battle_record->writable()) {
+    this->battle_record->add_command(BattleRecord::Event::Type::SERVER_DATA_COMMAND, data.data(), data.size());
+  }
+
+  if ((sender_c && (sender_c->version() == Version::GC_EP3_NTE)) || !header.mask_key) {
     (this->*handler)(sender_c, data);
   } else {
     string unmasked_data = data;
@@ -1786,8 +1837,7 @@ void Server::on_server_data_input(shared_ptr<Client> sender_c, const string& dat
 
 void Server::handle_CAx0B_mulligan_hand(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_RedrawInitialHand_Ep3_CAx0B>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "REDRAW");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "REDRAW");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -1819,8 +1869,7 @@ void Server::handle_CAx0B_mulligan_hand(shared_ptr<Client>, const string& data)
 
 void Server::handle_CAx0C_end_mulligan_phase(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_EndInitialRedrawPhase_Ep3_CAx0C>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "SETUP ADV 2");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "SETUP ADV 2");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -1901,8 +1950,7 @@ void Server::handle_CAx0D_end_non_action_phase(shared_ptr<Client>, const string&
 
 void Server::handle_CAx0E_discard_card_from_hand(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_DiscardCardFromHand_Ep3_CAx0E>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "DISCARD");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "DISCARD");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -1987,8 +2035,7 @@ void Server::handle_CAx0F_set_card_from_hand(shared_ptr<Client>, const string& d
 
 void Server::handle_CAx10_move_fc_to_location(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_MoveFieldCharacter_Ep3_CAx10>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "MOVE");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "MOVE");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -2028,8 +2075,7 @@ void Server::handle_CAx10_move_fc_to_location(shared_ptr<Client>, const string&
 
 void Server::handle_CAx11_enqueue_attack_or_defense(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_EnqueueAttackOrDefense_Ep3_CAx11>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "ENQUEUE ACT");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "ENQUEUE ACT");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -2067,8 +2113,7 @@ void Server::handle_CAx11_enqueue_attack_or_defense(shared_ptr<Client>, const st
 
 void Server::handle_CAx12_end_attack_list(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_EndAttackList_Ep3_CAx12>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "END ATK LIST");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "END ATK LIST");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -2091,22 +2136,42 @@ void Server::handle_CAx12_end_attack_list(shared_ptr<Client>, const string& data
 }
 
 template <typename CmdT>
-void Server::handle_CAx13_update_map_during_setup_t(shared_ptr<Client>, const string& data) {
+void Server::handle_CAx13_update_map_during_setup_t(shared_ptr<Client> c, const string& data) {
   const auto& in_cmd = check_size_t<CmdT>(data);
-  this->send_debug_command_received_message(
-      in_cmd.header.subsubcommand, "UPDATE MAP");
+  this->send_debug_command_received_message(in_cmd.header.subsubcommand, "UPDATE MAP");
 
   if (!this->battle_in_progress &&
       (this->setup_phase == SetupPhase::REGISTRATION) &&
       (this->map_and_rules->num_players == 0) &&
       (this->registration_phase != RegistrationPhase::REGISTERED) &&
       (this->registration_phase != RegistrationPhase::BATTLE_STARTED)) {
-    // newserv's extended rules are stored in unused parts of the Rules struct,
-    // and clients will probably overwrite them with zeroes if we allow them to.
-    // So, we preserve the extended rules manually here.
-    uint8_t def_dice_range = this->map_and_rules->rules.def_dice_range;
     *this->map_and_rules = in_cmd.map_and_rules_state;
-    this->map_and_rules->rules.def_dice_range = def_dice_range;
+    // The client will likely send incorrect values for the extended rules (or
+    // in the case of NTE, no values at all, since the Rules structure is
+    // smaller). So, use the values from the last chosen map if applicable, or
+    // the values from the $dicerange command if available.
+    uint8_t language = c ? c->language() : 1;
+    const Rules* map_rules = this->last_chosen_map ? &this->last_chosen_map->version(language)->map->default_rules : nullptr;
+    auto& server_rules = this->map_and_rules->rules;
+    // NTE can specify the DEF dice value range in its Rules struct, so we use
+    // that unless the map or $dicerange overrides it.
+    server_rules.def_dice_value_range = (map_rules && (map_rules->def_dice_value_range != 0xFF))
+        ? map_rules->def_dice_value_range
+        : (this->def_dice_value_range_override != 0xFF)
+        ? this->def_dice_value_range_override
+        : this->options.is_nte()
+        ? server_rules.def_dice_value_range
+        : 0;
+    server_rules.atk_dice_value_range_2v1 = (map_rules && (map_rules->atk_dice_value_range_2v1 != 0xFF))
+        ? map_rules->atk_dice_value_range_2v1
+        : (this->atk_dice_value_range_2v1_override != 0xFF)
+        ? this->atk_dice_value_range_2v1_override
+        : 0;
+    server_rules.def_dice_value_range_2v1 = (map_rules && (map_rules->def_dice_value_range_2v1 != 0xFF))
+        ? map_rules->def_dice_value_range_2v1
+        : (this->def_dice_value_range_2v1_override != 0xFF)
+        ? this->def_dice_value_range_2v1_override
+        : 0;
 
     // If this match is part of a tournament, ignore the rules sent by the
     // client and use the tournament rules instead.
@@ -2143,8 +2208,7 @@ void Server::handle_CAx13_update_map_during_setup(shared_ptr<Client> c, const st
 
 void Server::handle_CAx14_update_deck_during_setup(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_SetPlayerDeck_Ep3_CAx14>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "UPDATE DECK");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "UPDATE DECK");
 
   if (!this->battle_in_progress) {
     if ((this->setup_phase == SetupPhase::REGISTRATION) &&
@@ -2165,7 +2229,7 @@ void Server::handle_CAx14_update_deck_during_setup(shared_ptr<Client>, const str
         }
       }
       if (verify_error) {
-        throw runtime_error(string_printf("invalid deck: -0x%" PRIX32, verify_error));
+        throw runtime_error(phosg::string_printf("invalid deck: -0x%" PRIX32, verify_error));
       }
       if (!this->options.is_nte() && !(this->options.behavior_flags & BehaviorFlag::SKIP_D1_D2_REPLACE)) {
         this->ruler_server->replace_D1_D2_rank_cards_with_Attack(entry.card_ids);
@@ -2190,8 +2254,7 @@ void Server::handle_CAx14_update_deck_during_setup(shared_ptr<Client>, const str
 
 void Server::handle_CAx15_unused_hard_reset_server_state(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_HardResetServerState_Ep3_CAx15>(data);
-  this->send_debug_command_received_message(
-      in_cmd.header.subsubcommand, "HARD RESET");
+  this->send_debug_command_received_message(in_cmd.header.subsubcommand, "HARD RESET");
 
   // In the original implementation, this command recreates the server object.
   // This is possible because the dispatch function is not part of the server
@@ -2208,8 +2271,7 @@ void Server::handle_CAx15_unused_hard_reset_server_state(shared_ptr<Client>, con
 
 void Server::handle_CAx1B_update_player_name(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_SetPlayerName_Ep3_CAx1B>(data);
-  this->send_debug_command_received_message(
-      in_cmd.entry.client_id, in_cmd.header.subsubcommand, "UPDATE NAME");
+  this->send_debug_command_received_message(in_cmd.entry.client_id, in_cmd.header.subsubcommand, "UPDATE NAME");
 
   if (in_cmd.entry.client_id < 4) {
     if (!this->is_registration_complete()) {
@@ -2240,8 +2302,7 @@ void Server::handle_CAx1B_update_player_name(shared_ptr<Client>, const string& d
 
 void Server::handle_CAx1D_start_battle(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_StartBattle_Ep3_CAx1D>(data);
-  this->send_debug_command_received_message(
-      in_cmd.header.subsubcommand, "START BATTLE");
+  this->send_debug_command_received_message(in_cmd.header.subsubcommand, "START BATTLE");
 
   if (!this->battle_in_progress) {
     bool is_nte = this->options.is_nte();
@@ -2266,11 +2327,12 @@ void Server::handle_CAx1D_start_battle(shared_ptr<Client>, const string& data) {
     }
 
     if (should_start) {
+      if (this->battle_record && this->battle_record->writable()) {
+        this->battle_record->set_battle_start_timestamp();
+      }
+
       auto l = this->lobby.lock();
       if (l) {
-        if (l->battle_record) {
-          l->battle_record->set_battle_start_timestamp();
-        }
         // Note: Sega's implementation doesn't set EX results values here; they
         // did it at game join time instead. We do it here for code simplicity.
         if ((l->base_version != Version::GC_EP3_NTE) && l->ep3_ex_result_values) {
@@ -2286,8 +2348,7 @@ void Server::handle_CAx1D_start_battle(shared_ptr<Client>, const string& data) {
 
 void Server::handle_CAx21_end_battle(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_EndBattle_Ep3_CAx21>(data);
-  this->send_debug_command_received_message(
-      in_cmd.header.subsubcommand, "END BATTLE");
+  this->send_debug_command_received_message(in_cmd.header.subsubcommand, "END BATTLE");
   if (this->setup_phase == SetupPhase::BATTLE_ENDED) {
     this->battle_finished = true;
 
@@ -2301,8 +2362,7 @@ void Server::handle_CAx21_end_battle(shared_ptr<Client>, const string& data) {
 
 void Server::handle_CAx28_end_defense_list(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_EndDefenseList_Ep3_CAx28>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "END DEF LIST");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "END DEF LIST");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -2362,8 +2422,7 @@ void Server::handle_CAx34_subtract_ally_atk_points(shared_ptr<Client>, const str
   const auto& in_cmd = check_size_t<G_PhotonBlastRequest_Ep3_CAx34>(data);
 
   uint8_t card_ref_client_id = client_id_for_card_ref(in_cmd.card_ref);
-  this->send_debug_command_received_message(
-      card_ref_client_id, in_cmd.header.subsubcommand, "SUB ALLY ATK");
+  this->send_debug_command_received_message(card_ref_client_id, in_cmd.header.subsubcommand, "SUB ALLY ATK");
 
   if (card_ref_client_id >= 4) {
     return;
@@ -2413,8 +2472,7 @@ void Server::handle_CAx34_subtract_ally_atk_points(shared_ptr<Client>, const str
               attacker_card->card_flags |= 0x400;
               attacker_card->player_state()->send_6xB4x04_if_needed();
             }
-            uint16_t card_ref = this->send_6xB4x06_if_card_ref_invalid(
-                pa.original_attacker_card_ref, 9);
+            uint16_t card_ref = this->send_6xB4x06_if_card_ref_invalid(pa.original_attacker_card_ref, 9);
             auto orig_attacker_card = this->card_for_set_card_ref(card_ref);
             auto target_card = this->card_for_set_card_ref(pa.target_card_refs[0]);
             if (orig_attacker_card && target_card) {
@@ -2437,8 +2495,7 @@ void Server::handle_CAx34_subtract_ally_atk_points(shared_ptr<Client>, const str
 
 void Server::handle_CAx37_client_ready_to_advance_from_starter_roll_phase(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_AdvanceFromStartingRollsPhase_Ep3_CAx37>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "SETUP ADV 1");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "SETUP ADV 1");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -2474,8 +2531,7 @@ void Server::handle_CAx3A_time_limit_expired(shared_ptr<Client>, const string& d
 
 void Server::handle_CAx40_map_list_request(shared_ptr<Client> sender_c, const string& data) {
   const auto& in_cmd = check_size_t<G_MapListRequest_Ep3_CAx40>(data);
-  this->send_debug_command_received_message(
-      in_cmd.header.subsubcommand, "MAP LIST");
+  this->send_debug_command_received_message(in_cmd.header.subsubcommand, "MAP LIST");
 
   auto l = this->lobby.lock();
   if (!l) {
@@ -2486,7 +2542,7 @@ void Server::handle_CAx40_map_list_request(shared_ptr<Client> sender_c, const st
   uint8_t language = sender_c ? sender_c->language() : 1;
   const auto& list_data = this->options.map_index->get_compressed_list(num_players, language);
 
-  StringWriter w;
+  phosg::StringWriter w;
   uint32_t subcommand_size = (list_data.size() + sizeof(G_MapList_Ep3_6xB6x40) + 3) & (~3);
   w.put<G_MapList_Ep3_6xB6x40>(G_MapList_Ep3_6xB6x40{{{{0xB6, 0, 0}, subcommand_size}, 0x40, {}}, list_data.size(), 0});
   w.write(list_data);
@@ -2499,6 +2555,10 @@ void Server::handle_CAx40_map_list_request(shared_ptr<Client> sender_c, const st
 }
 
 void Server::send_6xB6x41_to_all_clients() const {
+  if (!this->last_chosen_map) {
+    throw logic_error("cannot send 6xB4x41 without a map chosen");
+  }
+
   auto l = this->lobby.lock();
   if (l) {
     vector<string> map_commands_by_language;
@@ -2525,13 +2585,13 @@ void Server::send_6xB6x41_to_all_clients() const {
       }
     }
 
-    if (l->battle_record && l->battle_record->writable()) {
+    if (this->battle_record && this->battle_record->writable()) {
       // TODO: It's not great that we just pick the first one; ideally we'd put
       // all of them in the recording and send the appropriate one to the client
       // in the playback lobby
       for (string& data : map_commands_by_language) {
         if (!data.empty()) {
-          l->battle_record->add_command(
+          this->battle_record->add_command(
               BattleRecord::Event::Type::BATTLE_COMMAND, std::move(data));
           break;
         }
@@ -2546,17 +2606,16 @@ void Server::send_6xB6x41_to_all_clients() const {
 
 void Server::handle_CAx41_map_request(shared_ptr<Client>, const string& data) {
   const auto& cmd = check_size_t<G_MapDataRequest_Ep3_CAx41>(data);
-  this->send_debug_command_received_message(
-      cmd.header.subsubcommand, "MAP DATA");
-
-  this->last_chosen_map = this->options.map_index->for_number(cmd.map_number);
-  this->send_6xB6x41_to_all_clients();
+  this->send_debug_command_received_message(cmd.header.subsubcommand, "MAP DATA");
+  if (!this->options.tournament || (this->options.tournament->get_map()->map_number == cmd.map_number)) {
+    this->last_chosen_map = this->options.map_index->for_number(cmd.map_number);
+    this->send_6xB6x41_to_all_clients();
+  }
 }
 
 void Server::handle_CAx48_end_turn(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_EndTurn_Ep3_CAx48>(data);
-  this->send_debug_command_received_message(
-      in_cmd.client_id, in_cmd.header.subsubcommand, "END TURN");
+  this->send_debug_command_received_message(in_cmd.client_id, in_cmd.header.subsubcommand, "END TURN");
   if (in_cmd.client_id >= 4) {
     throw runtime_error("invalid client ID");
   }
@@ -2573,8 +2632,7 @@ void Server::handle_CAx48_end_turn(shared_ptr<Client>, const string& data) {
 
 void Server::handle_CAx49_card_counts(shared_ptr<Client>, const string& data) {
   const auto& in_cmd = check_size_t<G_CardCounts_Ep3_CAx49>(data);
-  this->send_debug_command_received_message(
-      in_cmd.header.sender_client_id, in_cmd.header.subsubcommand, "CARD COUNTS");
+  this->send_debug_command_received_message(in_cmd.header.sender_client_id, in_cmd.header.subsubcommand, "CARD COUNTS");
 
   // Note: Sega's implmentation completely ignores this command. This
   // implementation is not based on the original code.
@@ -2713,11 +2771,9 @@ uint32_t Server::get_team_exp(uint8_t team_id) const {
   return this->team_exp[team_id];
 }
 
-uint32_t Server::send_6xB4x06_if_card_ref_invalid(
-    uint16_t card_ref, int16_t negative_value) {
+uint32_t Server::send_6xB4x06_if_card_ref_invalid(uint16_t card_ref, int16_t negative_value) {
   if (this->card_special) {
-    return this->card_special->send_6xB4x06_if_card_ref_invalid(
-        card_ref, -negative_value);
+    return this->card_special->send_6xB4x06_if_card_ref_invalid(card_ref, -negative_value);
   }
   return card_ref;
 }
@@ -2735,16 +2791,20 @@ void Server::unknown_8023EEF4() {
     auto card = this->attack_cards[this->unknown_a14];
     if (this->get_current_team_turn() == card->get_team_id()) {
       ActionState as = this->pending_attacks_with_cards[this->unknown_a14];
-      log.debug("card @%04hX #%04hX can attack", card->get_card_ref(), card->get_card_id());
-      string as_str = as.str();
-      log.debug("as: %s", as_str.c_str());
+      if (log.should_log(phosg::LogLevel::DEBUG)) {
+        log.debug("card @%04hX #%04hX can attack", card->get_card_ref(), card->get_card_id());
+        string as_str = as.str(this->shared_from_this());
+        log.debug("as: %s", as_str.c_str());
+      }
       if (is_nte) {
         this->replace_targets_due_to_destruction_nte(&as);
       } else {
         this->replace_targets_due_to_destruction_or_conditions(&as);
       }
-      as_str = as.str();
-      log.debug("as after target replacement: %s", as_str.c_str());
+      if (log.should_log(phosg::LogLevel::DEBUG)) {
+        string as_str = as.str(this->shared_from_this());
+        log.debug("as after target replacement: %s", as_str.c_str());
+      }
       if (this->any_target_exists_for_attack(as)) {
         log.debug("as is valid");
         break;
@@ -2761,8 +2821,8 @@ void Server::unknown_8023EEF4() {
     log.debug("a14 (%" PRIu32 ") < num_pending_attacks_with_cards (%" PRIu32 ")", this->unknown_a14, this->num_pending_attacks_with_cards);
     this->defense_list_ended_for_client.clear(false);
 
-    G_SetActionState_Ep3_6xB4x29 cmd;
-    cmd.unknown_a1 = this->unknown_a14;
+    G_UpdateAttackTargets_Ep3_6xB4x29 cmd;
+    cmd.attack_number = this->unknown_a14;
     cmd.state = this->pending_attacks_with_cards[this->unknown_a14];
     if (is_nte) {
       this->replace_targets_due_to_destruction_nte(&cmd.state);
@@ -2772,7 +2832,7 @@ void Server::unknown_8023EEF4() {
     ActionState as = cmd.state;
     this->send(cmd);
 
-    this->card_special->unknown_8024AAB8(as);
+    this->card_special->apply_effects_after_attack_target_resolution(as);
 
     if (!is_nte) {
       this->attack_cards[this->unknown_a14]->compute_action_chain_results(1, 0);
@@ -2856,7 +2916,7 @@ void Server::replace_targets_due_to_destruction_nte(ActionState* as) {
     if (!target_card) {
       break;
     }
-    if ((target_card->card_flags & 2) ||
+    if (!(target_card->card_flags & 2) ||
         (target_card->get_definition()->def.type != CardType::ITEM) ||
         attacker_card->action_chain.check_flag(0x02)) {
       continue;

src/Episode3/Server.hh

@@ -9,6 +9,7 @@
 #include "../CommandFormats.hh"
 #include "../Text.hh"
 #include "AssistServer.hh"
+#include "BattleRecord.hh"
 #include "CardSpecial.hh"
 #include "MapState.hh"
 #include "PlayerState.hh"
@@ -71,7 +72,8 @@ public:
     std::shared_ptr<const CardIndex> card_index;
     std::shared_ptr<const MapIndex> map_index;
     uint32_t behavior_flags;
-    std::shared_ptr<PSOLFGEncryption> random_crypt;
+    std::shared_ptr<phosg::StringReader> opt_rand_stream;
+    std::shared_ptr<PSOLFGEncryption> opt_rand_crypt;
     std::shared_ptr<const Tournament> tournament;
     std::array<std::vector<uint16_t>, 5> trap_card_ids;
 
@@ -83,10 +85,10 @@ public:
   ~Server() noexcept(false);
   void init();
 
-  class StackLogger : public PrefixedLogger {
+  class StackLogger : public phosg::PrefixedLogger {
   public:
     StackLogger(const Server* s, const std::string& prefix);
-    StackLogger(const Server* s, const std::string& prefix, LogLevel min_level);
+    StackLogger(const Server* s, const std::string& prefix, phosg::LogLevel min_level);
     StackLogger(const StackLogger&) = delete;
     StackLogger(StackLogger&&);
     StackLogger& operator=(const StackLogger&) = delete;
@@ -99,6 +101,33 @@ public:
   StackLogger log_stack(const std::string& prefix) const;
   const StackLogger& log() const;
 
+  std::string debug_str_for_card_ref(uint16_t card_ref) const;
+  std::string debug_str_for_card_id(uint16_t card_id) const;
+  template <typename U16T>
+  std::string debug_str_for_card_refs(const U16T* refs, size_t count) const {
+    std::string ret = "[";
+    for (size_t z = 0; z < count; z++) {
+      if (refs[z] != 0xFFFF) {
+        std::string ref_str = this->debug_str_for_card_ref(refs[z]);
+        ret += phosg::string_printf("%zu:%s ", z, ref_str.c_str());
+      }
+    }
+    if (ret.size() > 1) {
+      ret.back() = ']'; // Replace the ' ' from the last added item
+    } else {
+      ret.push_back(']');
+    }
+    return ret;
+  }
+  template <typename U16T>
+  std::string debug_str_for_card_refs(const std::vector<U16T>& refs) const {
+    return this->debug_str_for_card_refs(refs.data(), refs.size());
+  }
+  template <typename U16T, size_t Count>
+  std::string debug_str_for_card_refs(const parray<U16T, Count>& refs) const {
+    return this->debug_str_for_card_refs(refs.data(), refs.size());
+  }
+
   int8_t get_winner_team_id() const;
 
   template <typename T>
@@ -164,6 +193,7 @@ public:
   uint8_t get_current_team_turn() const;
   std::shared_ptr<PlayerState> get_player_state(uint8_t client_id);
   std::shared_ptr<const PlayerState> get_player_state(uint8_t client_id) const;
+  uint32_t get_random_raw();
   uint32_t get_random(uint32_t max);
   float get_random_float_0_1();
   uint32_t get_round_num() const;
@@ -178,6 +208,7 @@ public:
   void send_set_card_updates_and_6xB4x04_if_needed();
   void set_battle_ended();
   void set_battle_started();
+  bool player_can_receive_dice_boost(uint8_t client_id) const;
   void set_client_id_ready_to_advance_phase(uint8_t client_id, BattlePhase battle_phase);
   void set_phase_after();
   void move_phase_before();
@@ -245,11 +276,15 @@ private:
 public:
   // These fields are not part of the original implementation
   std::weak_ptr<Lobby> lobby;
+  std::shared_ptr<BattleRecord> battle_record;
   bool has_lobby;
   Options options;
   std::shared_ptr<const MapIndex::Map> last_chosen_map;
   bool tournament_match_result_sent;
   uint8_t override_environment_number;
+  uint8_t def_dice_value_range_override;
+  uint8_t atk_dice_value_range_2v1_override;
+  uint8_t def_dice_value_range_2v1_override;
   mutable std::deque<StackLogger*> logger_stack;
 
   // These fields were originally contained in the TCardServerBase object
@@ -259,9 +294,10 @@ public:
     uint8_t is_cpu_player;
     PresenceEntry();
     void clear();
-  } __attribute__((packed));
+  } __packed_ws__(PresenceEntry, 3);
+
   std::shared_ptr<MapAndRulesState> map_and_rules;
-  std::shared_ptr<DeckEntry> deck_entries[4];
+  bcarray<std::shared_ptr<DeckEntry>, 4> deck_entries;
   parray<PresenceEntry, 4> presence_entries;
   uint8_t num_clients_present;
   parray<NameEntry, 4> name_entries;
@@ -280,7 +316,7 @@ public:
   RegistrationPhase registration_phase;
   ActionSubphase action_subphase;
   uint8_t current_team_turn2;
-  ActionState pending_attacks[0x20];
+  bcarray<ActionState, 0x20> pending_attacks;
   uint32_t num_pending_attacks;
   parray<uint8_t, 4> client_done_enqueuing_attacks;
   parray<uint8_t, 4> player_ready_to_end_phase;
@@ -296,8 +332,8 @@ public:
   std::array<std::shared_ptr<PlayerState>, 4> player_states;
   parray<uint32_t, 4> clients_done_in_mulligan_phase;
   uint32_t num_pending_attacks_with_cards;
-  std::shared_ptr<Card> attack_cards[0x20];
-  ActionState pending_attacks_with_cards[0x20];
+  bcarray<std::shared_ptr<Card>, 0x20> attack_cards;
+  bcarray<ActionState, 0x20> pending_attacks_with_cards;
   uint32_t unknown_a14;
   uint32_t unknown_a15;
   parray<uint32_t, 4> defense_list_ended_for_client;
@@ -315,7 +351,7 @@ public:
   parray<parray<parray<uint8_t, 2>, 8>, 5> trap_tile_locs;
   parray<parray<uint8_t, 2>, 0x10> trap_tile_locs_nte;
   size_t num_trap_tiles_nte;
-  ActionState pb_action_states[4];
+  bcarray<ActionState, 4> pb_action_states;
   parray<uint8_t, 4> has_done_pb;
   parray<parray<uint8_t, 4>, 4> has_done_pb_with_client;
   mutable uint32_t num_6xB4x06_commands_sent;

src/Episode3/Tournament.cc

@@ -9,18 +9,18 @@ using namespace std;
 
 namespace Episode3 {
 
-Tournament::PlayerEntry::PlayerEntry(uint32_t serial_number, const string& player_name)
-    : serial_number(serial_number),
+Tournament::PlayerEntry::PlayerEntry(uint32_t account_id, const string& player_name)
+    : account_id(account_id),
       player_name(player_name) {}
 
 Tournament::PlayerEntry::PlayerEntry(shared_ptr<Client> c)
-    : serial_number(c->license->serial_number),
+    : account_id(c->login->account->account_id),
       client(c),
       player_name(c->character()->disp.name.decode(c->language())) {}
 
 Tournament::PlayerEntry::PlayerEntry(
     shared_ptr<const COMDeckDefinition> com_deck)
-    : serial_number(0),
+    : account_id(0),
       com_deck(com_deck) {}
 
 bool Tournament::PlayerEntry::is_com() const {
@@ -28,7 +28,7 @@ bool Tournament::PlayerEntry::is_com() const {
 }
 
 bool Tournament::PlayerEntry::is_human() const {
-  return (this->serial_number != 0);
+  return (this->account_id != 0);
 }
 
 Tournament::Team::Team(
@@ -49,16 +49,16 @@ string Tournament::Team::str() const {
     num_com_players += player.is_com();
   }
 
-  string ret = string_printf("[Team/%zu %s %zuH/%zuC/%zuP name=%s pass=%s rounds=%zu",
+  string ret = phosg::string_printf("[Team/%zu %s %zuH/%zuC/%zuP name=%s pass=%s rounds=%zu",
       this->index, this->is_active ? "active" : "inactive",
       num_human_players, num_com_players, this->max_players, this->name.c_str(),
       this->password.c_str(), this->num_rounds_cleared);
   for (const auto& player : this->players) {
     if (player.is_human()) {
       if (player.player_name.empty()) {
-        ret += string_printf(" %08" PRIX32, player.serial_number);
+        ret += phosg::string_printf(" %08" PRIX32, player.account_id);
       } else {
-        ret += string_printf(" %08" PRIX32 " (%s)", player.serial_number, player.player_name.c_str());
+        ret += phosg::string_printf(" %08" PRIX32 " (%s)", player.account_id, player.player_name.c_str());
       }
     }
   }
@@ -81,12 +81,12 @@ void Tournament::Team::register_player(
   if (!tournament) {
     throw runtime_error("tournament has been deleted");
   }
-  if (!tournament->all_player_serial_numbers.emplace(c->license->serial_number).second) {
+  if (!tournament->all_player_account_ids.emplace(c->login->account->account_id).second) {
     throw runtime_error("player already registered in same tournament");
   }
 
   for (const auto& player : this->players) {
-    if (player.is_human() && (player.serial_number == c->license->serial_number)) {
+    if (player.is_human() && (player.account_id == c->login->account->account_id)) {
       throw logic_error("player already registered in team but not in tournament");
     }
   }
@@ -99,11 +99,11 @@ void Tournament::Team::register_player(
   }
 }
 
-bool Tournament::Team::unregister_player(uint32_t serial_number) {
+bool Tournament::Team::unregister_player(uint32_t account_id) {
   size_t index;
   for (index = 0; index < this->players.size(); index++) {
     if (this->players[index].is_human() &&
-        (this->players[index].serial_number == serial_number)) {
+        (this->players[index].account_id == account_id)) {
       break;
     }
   }
@@ -143,7 +143,7 @@ bool Tournament::Team::unregister_player(uint32_t serial_number) {
       // If the tournament has not started yet, just remove the player from the
       // team
     } else {
-      if (!tournament->all_player_serial_numbers.erase(serial_number)) {
+      if (!tournament->all_player_account_ids.erase(account_id)) {
         throw logic_error("player removed from team but not from tournament");
       }
     }
@@ -206,7 +206,7 @@ Tournament::Match::Match(
 
 string Tournament::Match::str() const {
   string winner_str = this->winner_team ? this->winner_team->str() : "(none)";
-  return string_printf("[Match round=%zu winner=%s]", this->round_num, winner_str.c_str());
+  return phosg::string_printf("[Match round=%zu winner=%s]", this->round_num, winner_str.c_str());
 }
 
 bool Tournament::Match::resolve_if_skippable() {
@@ -230,7 +230,7 @@ bool Tournament::Match::resolve_if_skippable() {
   // entirely and just make one team advance arbitrarily (note that this also
   // handles the case where both preceding winner teams are empty)
   if (!winner_a->has_any_human_players() && !winner_b->has_any_human_players()) {
-    this->set_winner_team((random_object<uint8_t>() & 1) ? winner_b : winner_a);
+    this->set_winner_team((phosg::random_object<uint8_t>() & 1) ? winner_b : winner_a);
     return true;
   }
 
@@ -318,7 +318,7 @@ Tournament::Tournament(
     const Rules& rules,
     size_t num_teams,
     uint8_t flags)
-    : log(string_printf("[Tournament:%s] ", name.c_str())),
+    : log(phosg::string_printf("[Tournament:%s] ", name.c_str())),
       map_index(map_index),
       com_deck_index(com_deck_index),
       name(name),
@@ -342,8 +342,8 @@ Tournament::Tournament(
 Tournament::Tournament(
     shared_ptr<const MapIndex> map_index,
     shared_ptr<const COMDeckIndex> com_deck_index,
-    const JSON& json)
-    : log(string_printf("[Tournament:%s] ", json.get_string("name").c_str())),
+    const phosg::JSON& json)
+    : log(phosg::string_printf("[Tournament:%s] ", json.get_string("name").c_str())),
       map_index(map_index),
       com_deck_index(com_deck_index),
       source_json(json),
@@ -371,13 +371,13 @@ void Tournament::init() {
       team_index_to_rounds_cleared.emplace_back(team_json->get_int("num_rounds_cleared"));
       for (const auto& player_json : team_json->get_list("player_specs")) {
         if (player_json->is_list()) {
-          uint32_t serial_number = player_json->at(0).as_int();
-          team->players.emplace_back(serial_number, player_json->at(1).as_string());
-          this->all_player_serial_numbers.emplace(serial_number);
+          uint32_t account_id = player_json->at(0).as_int();
+          team->players.emplace_back(account_id, player_json->at(1).as_string());
+          this->all_player_account_ids.emplace(account_id);
         } else if (player_json->is_int()) {
-          uint32_t serial_number = player_json->as_int();
-          team->players.emplace_back(serial_number);
-          this->all_player_serial_numbers.emplace(serial_number);
+          uint32_t account_id = player_json->as_int();
+          team->players.emplace_back(account_id);
+          this->all_player_account_ids.emplace(account_id);
         } else if (player_json->is_string()) {
           team->players.emplace_back(this->com_deck_index->deck_for_name(player_json->as_string()));
         } else {
@@ -504,22 +504,22 @@ void Tournament::create_bracket_matches() {
   this->final_match = current_round_matches.at(0);
 }
 
-JSON Tournament::json() const {
-  auto teams_list = JSON::list();
+phosg::JSON Tournament::json() const {
+  auto teams_list = phosg::JSON::list();
   for (auto team : this->teams) {
-    auto players_list = JSON::list();
+    auto players_list = phosg::JSON::list();
     for (const auto& player : team->players) {
       if (player.is_human()) {
         if (!player.player_name.empty()) {
-          players_list.emplace_back(JSON::list({player.serial_number, player.player_name}));
+          players_list.emplace_back(phosg::JSON::list({player.account_id, player.player_name}));
         } else {
-          players_list.emplace_back(player.serial_number);
+          players_list.emplace_back(player.account_id);
         }
       } else {
         players_list.emplace_back(player.com_deck->deck_name);
       }
     }
-    teams_list.emplace_back(JSON::dict({
+    teams_list.emplace_back(phosg::JSON::dict({
         {"max_players", team->max_players},
         {"player_specs", std::move(players_list)},
         {"name", team->name},
@@ -527,7 +527,7 @@ JSON Tournament::json() const {
         {"num_rounds_cleared", team->num_rounds_cleared},
     }));
   }
-  return JSON::dict({
+  return phosg::JSON::dict({
       {"name", this->name},
       {"map_number", this->map->map_number},
       {"rules", this->rules.json()},
@@ -571,25 +571,25 @@ shared_ptr<Tournament::Match> Tournament::get_final_match() const {
   return this->final_match;
 }
 
-shared_ptr<Tournament::Team> Tournament::team_for_serial_number(
-    uint32_t serial_number) const {
-  if (!this->all_player_serial_numbers.count(serial_number)) {
+shared_ptr<Tournament::Team> Tournament::team_for_account_id(
+    uint32_t account_id) const {
+  if (!this->all_player_account_ids.count(account_id)) {
     return nullptr;
   }
 
   for (auto team : this->teams) {
     for (const auto& player : team->players) {
-      if (player.serial_number == serial_number) {
+      if (player.account_id == account_id) {
         return team->is_active ? team : nullptr;
       }
     }
   }
 
-  throw logic_error("serial number registered in tournament but not in any team");
+  throw logic_error("account ID registered in tournament but not in any team");
 }
 
-const set<uint32_t>& Tournament::get_all_player_serial_numbers() const {
-  return this->all_player_serial_numbers;
+const set<uint32_t>& Tournament::get_all_player_account_ids() const {
+  return this->all_player_account_ids;
 }
 
 void Tournament::start() {
@@ -649,7 +649,7 @@ void Tournament::start() {
   if (this->flags & Flag::SHUFFLE_ENTRIES) {
     // Shuffle all the tournament entries
     for (size_t z = this->teams.size(); z > 0; z--) {
-      size_t index = random_object<uint32_t>() % z;
+      size_t index = phosg::random_object<uint32_t>() % z;
       if (index != z - 1) {
         this->teams[z - 1].swap(this->teams[index]);
       }
@@ -665,7 +665,7 @@ void Tournament::start() {
     auto m = this->zero_round_matches[z];
     auto t = m->winner_team;
     if (t->name.empty()) {
-      t->name = has_com_teams ? string_printf("COM:%zu", z) : "(no entrant)";
+      t->name = has_com_teams ? phosg::string_printf("COM:%zu", z) : "(no entrant)";
     }
     for (const auto& player : t->players) {
       if (player.is_com()) {
@@ -791,16 +791,16 @@ TournamentIndex::TournamentIndex(
     return;
   }
 
-  JSON json;
+  phosg::JSON json;
   try {
-    json = JSON::parse(load_file(this->state_filename));
-  } catch (const cannot_open_file&) {
-    json = JSON::list();
+    json = phosg::JSON::parse(phosg::load_file(this->state_filename));
+  } catch (const phosg::cannot_open_file&) {
+    json = phosg::JSON::list();
   }
 
   if (json.is_list()) {
     if (json.size() > 0x20) {
-      throw runtime_error("tournament JSON list length is incorrect");
+      throw runtime_error("tournament phosg::JSON list length is incorrect");
     }
     for (size_t z = 0; z < min<size_t>(json.size(), 0x20); z++) {
       if (!json.at(z).is_null()) {
@@ -815,13 +815,13 @@ TournamentIndex::TournamentIndex(
     }
   } else if (json.is_dict()) {
     if (json.size() > 0x20) {
-      throw runtime_error("tournament JSON dict length is incorrect");
+      throw runtime_error("tournament phosg::JSON dict length is incorrect");
     }
     for (const auto& it : json.as_dict()) {
       auto tourn = make_shared<Tournament>(this->map_index, this->com_deck_index, *it.second);
       tourn->init();
       if (!this->name_to_tournament.emplace(tourn->get_name(), tourn).second) {
-        // This is logic_error instead of runtime_error because JSON dicts are
+        // This is logic_error instead of runtime_error because phosg::JSON dicts are
         // supposed to already have unique keys
         throw logic_error("multiple tournaments have the same name: " + tourn->get_name());
       }
@@ -829,7 +829,7 @@ TournamentIndex::TournamentIndex(
       this->menu_item_id_to_tournament.emplace_back(tourn);
     }
   } else {
-    throw runtime_error("tournament state root JSON is not a list or dict");
+    throw runtime_error("tournament state root phosg::JSON is not a list or dict");
   }
 }
 
@@ -838,11 +838,11 @@ void TournamentIndex::save() const {
     return;
   }
 
-  auto json = JSON::dict();
+  auto json = phosg::JSON::dict();
   for (const auto& it : this->name_to_tournament) {
     json.emplace(it.second->get_name(), it.second->json());
   }
-  save_file(this->state_filename, json.serialize(JSON::SerializeOption::FORMAT | JSON::SerializeOption::HEX_INTEGERS | JSON::SerializeOption::ESCAPE_CONTROLS_ONLY));
+  phosg::save_file(this->state_filename, json.serialize(phosg::JSON::SerializeOption::FORMAT | phosg::JSON::SerializeOption::HEX_INTEGERS | phosg::JSON::SerializeOption::ESCAPE_CONTROLS_ONLY));
 }
 
 shared_ptr<Tournament> TournamentIndex::create_tournament(
@@ -896,10 +896,10 @@ bool TournamentIndex::delete_tournament(const string& name) {
   return true;
 }
 
-shared_ptr<Tournament::Team> TournamentIndex::team_for_serial_number(uint32_t serial_number) const {
+shared_ptr<Tournament::Team> TournamentIndex::team_for_account_id(uint32_t account_id) const {
   for (const auto& it : this->name_to_tournament) {
     const auto& tourn = it.second;
-    auto team = tourn->team_for_serial_number(serial_number);
+    auto team = tourn->team_for_account_id(account_id);
     if (team) {
       return team;
     }
@@ -912,11 +912,11 @@ void TournamentIndex::link_client(shared_ptr<Client> c) {
     return;
   }
 
-  auto team = this->team_for_serial_number(c->license->serial_number);
+  auto team = this->team_for_account_id(c->login->account->account_id);
   auto tourn = team ? team->tournament.lock() : nullptr;
   if (team && team->is_active && tourn) {
     for (auto& player : team->players) {
-      if (player.serial_number == c->license->serial_number) {
+      if (player.account_id == c->login->account->account_id) {
         c->ep3_tournament_team = team;
         player.client = c;
         if (c->version() == Version::GC_EP3) {

src/Episode3/Tournament.hh

@@ -33,23 +33,23 @@ public:
   };
 
   struct PlayerEntry {
-    // Invariant: (serial_number == 0) != (com_deck == nullptr)
+    // Invariant: (account_id == 0) != (com_deck == nullptr)
     // (that is, exactly one of the following must be valid)
-    uint32_t serial_number;
+    uint32_t account_id;
     std::shared_ptr<const COMDeckDefinition> com_deck;
 
-    // client is valid if serial_number is nonzero and the client is connected
+    // client is valid if account_id is nonzero and the client is connected
     std::weak_ptr<Client> client;
     std::string player_name; // Not used for COM decks
 
-    explicit PlayerEntry(uint32_t serial_number, const std::string& player_name = "");
+    explicit PlayerEntry(uint32_t account_id, const std::string& player_name = "");
     explicit PlayerEntry(std::shared_ptr<Client> c);
     explicit PlayerEntry(std::shared_ptr<const COMDeckDefinition> com_deck);
 
     bool is_com() const;
     bool is_human() const;
 
-    JSON json() const;
+    phosg::JSON json() const;
   };
 
   struct Team : public std::enable_shared_from_this<Team> {
@@ -73,7 +73,7 @@ public:
         std::shared_ptr<Client> c,
         const std::string& team_name,
         const std::string& password);
-    bool unregister_player(uint32_t serial_number);
+    bool unregister_player(uint32_t account_id);
 
     bool has_any_human_players() const;
     size_t num_human_players() const;
@@ -115,11 +115,11 @@ public:
   Tournament(
       std::shared_ptr<const MapIndex> map_index,
       std::shared_ptr<const COMDeckIndex> com_deck_index,
-      const JSON& json);
+      const phosg::JSON& json);
   ~Tournament() = default;
   void init();
 
-  JSON json() const;
+  phosg::JSON json() const;
 
   inline const std::string& get_name() const {
     return this->name;
@@ -152,8 +152,8 @@ public:
   std::shared_ptr<Team> get_winner_team() const;
   std::shared_ptr<Match> next_match_for_team(std::shared_ptr<Team> team) const;
   std::shared_ptr<Match> get_final_match() const;
-  std::shared_ptr<Team> team_for_serial_number(uint32_t serial_number) const;
-  const std::set<uint32_t>& get_all_player_serial_numbers() const;
+  std::shared_ptr<Team> team_for_account_id(uint32_t account_id) const;
+  const std::set<uint32_t>& get_all_player_account_ids() const;
 
   void start();
 
@@ -165,11 +165,11 @@ public:
 private:
   void create_bracket_matches();
 
-  PrefixedLogger log;
+  phosg::PrefixedLogger log;
 
   std::shared_ptr<const MapIndex> map_index;
   std::shared_ptr<const COMDeckIndex> com_deck_index;
-  JSON source_json;
+  phosg::JSON source_json;
   std::string name;
   std::shared_ptr<const MapIndex::Map> map;
   Rules rules;
@@ -178,7 +178,7 @@ private:
   State current_state;
   uint32_t menu_item_id;
 
-  std::set<uint32_t> all_player_serial_numbers;
+  std::set<uint32_t> all_player_account_ids;
   std::unordered_set<std::shared_ptr<Match>> pending_matches;
 
   // This vector contains all teams in the original starting order of the
@@ -231,7 +231,7 @@ public:
       uint8_t flags);
   bool delete_tournament(const std::string& name);
 
-  std::shared_ptr<Tournament::Team> team_for_serial_number(uint32_t serial_number) const;
+  std::shared_ptr<Tournament::Team> team_for_account_id(uint32_t account_id) const;
 
   void link_client(std::shared_ptr<Client> c);
   void link_all_clients(std::shared_ptr<ServerState> s);

src/EventUtils.cc

@@ -0,0 +1,43 @@
+#include "EventUtils.hh"
+
+#include <event2/event.h>
+
+#include <deque>
+#include <functional>
+#include <memory>
+#include <stdexcept>
+
+static void dispatch_forward_to_event_thread(evutil_socket_t, short, void* ctx) {
+  auto* fn = reinterpret_cast<std::function<void()>*>(ctx);
+  (*fn)();
+  delete fn;
+}
+
+void forward_to_event_thread(std::shared_ptr<struct event_base> base, std::function<void()>&& fn) {
+  struct timeval tv = {0, 0};
+  std::function<void()>* new_fn = new std::function<void()>(std::move(fn));
+  event_base_once(base.get(), -1, EV_TIMEOUT, dispatch_forward_to_event_thread, new_fn, &tv);
+}
+
+template <>
+void call_on_event_thread<void>(std::shared_ptr<struct event_base> base, std::function<void()>&& compute) {
+  bool succeeded = false;
+  std::string exc_what;
+  std::mutex ret_lock;
+  std::condition_variable ret_cv;
+  std::unique_lock<std::mutex> g(ret_lock);
+  forward_to_event_thread(base, [&]() -> void {
+    std::lock_guard<std::mutex> g(ret_lock);
+    try {
+      compute();
+      succeeded = true;
+    } catch (const std::exception& e) {
+      exc_what = e.what();
+    }
+    ret_cv.notify_one();
+  });
+  ret_cv.wait(g);
+  if (!succeeded) {
+    throw std::runtime_error(exc_what);
+  }
+}