fix initial lobby position commands across NTE/proto boundary

Should probably ask Flycast to do this automatically with zero values.

.gitignore

@@ -15,13 +15,15 @@ Testing
 
 # Files modified by the user and/or server that don't have defaults
 system/config.json
+system/ep3/battle-records/*.mzrd
 system/ep3/tournament-state.json
-system/ep3/maps-free/*.bind
-system/ep3/maps-quest/*.bind
 system/licenses.nsi
-system/players/player_*
-system/players/account_*
-system/players/bank_*
+system/licenses/*.json
+system/players/*.psochar
+system/players/*.psosys
+system/players/*.psocard
+system/players/*.nsc
+system/players/*.nsa
 system/patch-pc/.metadata-cache.json
 system/patch-bb/.metadata-cache.json
 

CMakeLists.txt

@@ -33,13 +33,15 @@ set (LIBEVENT_LIBRARIES
         ${LIBEVENT_CORE})
 
 find_package(phosg REQUIRED)
+find_package(Iconv REQUIRED)
 find_package(resource_file QUIET)
 
 
 
 # Executable definition
 
-add_executable(newserv
+set(SOURCES
+    src/AFSArchive.cc
     src/BattleParamsIndex.cc
     src/BMLArchive.cc
     src/CatSession.cc
@@ -48,6 +50,7 @@ add_executable(newserv
     src/Client.cc
     src/CommonItemSet.cc
     src/Compression.cc
+    src/DCSerialNumbers.cc
     src/DNSServer.cc
     src/EnemyType.cc
     src/Episode3/AssistServer.cc
@@ -65,10 +68,12 @@ add_executable(newserv
     src/FileContentsCache.cc
     src/FunctionCompiler.cc
     src/GSLArchive.cc
+    src/GVMEncoder.cc
     src/IPFrameInfo.cc
     src/IPStackSimulator.cc
     src/ItemCreator.cc
     src/ItemData.cc
+    src/ItemNameIndex.cc
     src/ItemParameterTable.cc
     src/Items.cc
     src/LevelTable.cc
@@ -81,7 +86,7 @@ add_executable(newserv
     src/NetworkAddresses.cc
     src/PatchFileIndex.cc
     src/Player.cc
-    src/Product.cc
+    src/PlayerSubordinates.cc
     src/ProxyCommands.cc
     src/ProxyServer.cc
     src/PSOEncryption.cc
@@ -100,11 +105,21 @@ add_executable(newserv
     src/ServerState.cc
     src/Shell.cc
     src/StaticGameData.cc
+    src/TeamIndex.cc
     src/Text.cc
+    src/TextArchive.cc
+    src/UnicodeTextSet.cc
     src/Version.cc
+    src/WordSelectTable.cc
 )
-target_include_directories(newserv PUBLIC ${LIBEVENT_INCLUDE_DIR})
-target_link_libraries(newserv phosg ${LIBEVENT_LIBRARIES} pthread)
+
+if(resource_file_FOUND)
+    set(SOURCES ${SOURCES} src/ARCodeTranslator.cc)
+endif()
+
+add_executable(newserv ${SOURCES})
+target_include_directories(newserv PUBLIC ${LIBEVENT_INCLUDE_DIR} ${Iconv_INCLUDE_DIRS})
+target_link_libraries(newserv phosg ${LIBEVENT_LIBRARIES} ${Iconv_LIBRARIES} pthread)
 
 if(resource_file_FOUND)
     target_compile_definitions(newserv PUBLIC HAVE_RESOURCE_FILE)
@@ -120,33 +135,23 @@ endif()
 
 enable_testing()
 
-file(GLOB TestCases ${CMAKE_SOURCE_DIR}/tests/*.test.txt)
+file(GLOB LogTestCases ${CMAKE_SOURCE_DIR}/tests/*.test.txt)
 
-foreach(TestCase IN ITEMS ${TestCases})
+foreach(LogTestCase IN ITEMS ${LogTestCases})
     add_test(
-        NAME ${TestCase}
+        NAME ${LogTestCase}
         WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-        COMMAND ${CMAKE_BINARY_DIR}/newserv replay-log ${TestCase} --config=${CMAKE_SOURCE_DIR}/tests/config.json --require-password=11111111 --require-access-key=111111111111)
+        COMMAND ${CMAKE_BINARY_DIR}/newserv --replay-log=${LogTestCase} --config=${CMAKE_SOURCE_DIR}/tests/config.json)
 endforeach()
 
-add_test(
-    NAME "compression-prs"
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    COMMAND ${CMAKE_SOURCE_DIR}/tests/test-compression.sh prs ${CMAKE_BINARY_DIR}/newserv)
-add_test(
-    NAME "compression-bc0"
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    COMMAND ${CMAKE_SOURCE_DIR}/tests/test-compression.sh bc0 ${CMAKE_BINARY_DIR}/newserv)
+file(GLOB ScriptTestCases ${CMAKE_SOURCE_DIR}/tests/*.test.sh)
 
-add_test(
-    NAME "decode-vms"
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    COMMAND ${CMAKE_SOURCE_DIR}/tests/test-decode-vms.sh ${CMAKE_BINARY_DIR}/newserv)
-
-add_test(
-    NAME "decode-gci"
-    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    COMMAND ${CMAKE_SOURCE_DIR}/tests/test-decode-gci.sh ${CMAKE_BINARY_DIR}/newserv)
+foreach(ScriptTestCase IN ITEMS ${ScriptTestCases})
+    add_test(
+        NAME ${ScriptTestCase}
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+        COMMAND ${ScriptTestCase} ${CMAKE_BINARY_DIR}/newserv)
+endforeach()
 
 # Installation configuration
 

README.md

@@ -1,6 +1,6 @@
 # newserv <img align="right" src="s-newserv.png" />
 
-newserv is a game server and proxy for Phantasy Star Online (PSO).
+newserv is a game server, proxy, and reverse-engineering tool for Phantasy Star Online (PSO).
 
 This project includes code that was reverse-engineered by the community in ages long past, and has been included in many projects since then. It also includes some game data from Phantasy Star Online itself, which was originally created by Sega.
 
@@ -19,6 +19,7 @@ This project includes code that was reverse-engineered by the community in ages
 * How to connect
     * Connecting local clients
         * [PSO DC](#pso-dc)
+        * [PSO DC on Flycast](#pso-dc-on-flycast)
         * [PSO PC](#pso-pc)
         * [PSO GC on a real GameCube](#pso-gc-on-a-real-gamecube)
         * [PSO GC on Dolphin](#pso-gc-on-dolphin)
@@ -49,58 +50,34 @@ newserv is many things - a server, a proxy, an encryption and decryption tool, a
 
 With that said, I offer no guarantees on how or when this project will advance. Feel free to submit GitHub issues if you find bugs or have feature requests; I'd like to make the server as stable and complete as possible, but I can't promise that I'll respond to issues in a timely manner. If you feel like contributing to newserv yourself, pull requests are welcome as well.
 
-Current known issues / missing features / things to do:
-- Implement the rest of PSOBB. Major areas of work:
-    - Find any remaining mismatches in enemy IDs / experience
-    - Sale prices for non-rare weapons with specials are computed incorrectly when buying/selling at shops
-    - Replace enemy list, game episode, etc. with quest data when loading a quest
-    - Implement trade window
-    - Fix some edge cases on the BB proxy server (e.g. make sure Change Ship does the right thing, which is not the same as what it should do on other versions).
-- There is a function that encodes QST files, but there's no corresponding CLI option.
-- Figure out what controls BML file data segment alignment.
-- Extension data in inventories is not handled properly.
-- PSOX is not tested at all.
-- Find a way to silence audio in RunDOL.s. Some old DOLs don't reset audio systems at load time and it's annoying to hear the crash buzz when the GC hasn't actually crashed.
-- Implement private and overflow lobbies.
-- Enforce client-side size limits (e.g. for 60/62 commands) on the server side as well. (For 60/62 specifically, perhaps transform them to 6C/6D if needed.)
-- Encapsulate BB server-side random state and make replays deterministic.
-- Implement character and inventory replacement for battle and challenge modes.
-- Implement the C5 (battle/challenge records) command.
-- Implement choice search.
-- Episode 3 bugs
-    - Fix behavior when joining a spectator team after the beginning of a battle.
-    - Disconnecting during a match turns you into a COM if there are other humans in the match, even if the match is part of a tournament. This may be incorrect behavior for tournaments.
-    - Disconnecting during a tournament when there are no other humans in the match simply cancels the match (so it can be replayed) instead of forfeiting, which is almost certainly incorrect behavior. (Then again, no one likes losing tournaments to COMs...)
-    - Tournament deck restrictions aren't enforced when populating COMs at tournament start time. This can cause weird behavior if, for example, a COM deck contains assist cards and the tournament rules forbid them.
-    - There is a rare failure mode during battles that causes one of the clients to be disconnected.
-- Code style
-    - Add default values in all command structures (like we use for Episode 3 battle commands).
+See TODO.md for a list of known issues and future work.
 
 ## Compatibility
 
 newserv supports several versions of PSO. Specifically:
 | Version        | Login        | Lobbies      | Games        | Proxy        |
 |----------------|--------------|--------------|--------------|--------------|
-| DC Trial       | Yes (4)      | Yes (4)      | Yes (4)      | No           |
-| DC Prototype   | Yes (4)      | Yes (4)      | Yes (4)      | No           |
-| DC V1          | Yes (1)      | Yes          | Yes          | Yes          |
-| DC V2          | Yes (1)      | Yes          | Yes          | Yes          |
+| DC Trial       | Yes          | Yes          | Yes          | No           |
+| DC 11/2000     | Yes          | Yes          | Yes          | No           |
+| DC 12/2000     | Yes          | Yes          | Yes          | Yes          |
+| DC 01/2001     | Yes          | Yes          | Yes          | Yes          |
+| DC V1          | Yes          | Yes          | Yes          | Yes          |
+| DC 08/2001     | Untested (1) | Untested (1) | Untested (1) | Untested (1) |
+| DC V2          | Yes          | Yes          | Yes          | Yes          |
 | PC             | Yes          | Yes          | Yes          | Yes          |
-| GC Ep1&2 Trial | Untested (2) | Untested (2) | Untested (2) | Untested (2) |
+| GC Ep1&2 Trial | Untested (1) | Untested (1) | Untested (1) | Untested (1) |
 | GC Ep1&2       | Yes          | Yes          | Yes          | Yes          |
 | GC Ep1&2 Plus  | Yes          | Yes          | Yes          | Yes          |
-| GC Ep3 Trial   | Yes          | Yes          | Partial (5)  | Yes          |
+| GC Ep3 Trial   | Yes          | Yes          | Partial (3)  | Yes          |
 | GC Ep3         | Yes          | Yes          | Yes          | Yes          |
-| XBOX Ep1&2     | Untested (2) | Untested (2) | Untested (2) | Untested (2) |
-| BB (vanilla)   | Yes          | Yes          | Yes (3)      | Yes          |
-| BB (Tethealla) | Yes          | Yes          | Yes (3)      | Yes          |
+| Xbox Ep1&2     | Yes          | Yes          | Yes          | Yes          |
+| BB (vanilla)   | Yes          | Yes          | Yes (2)      | Yes          |
+| BB (Tethealla) | Yes          | Yes          | Yes (2)      | Yes          |
 
 *Notes:*
-1. *DC support has only been tested with the US versions of PSO DC. Other versions probably don't work, but will be easy to add support for. Please submit a GitHub issue if you have a non-US DC version, and can provide a log from a connection attempt.*
-2. *newserv's implementations of these versions are based on disassembly of the client executables and have never been tested.*
-3. *BB games are mostly playable, but there are still some unimplemented features (for example, some quests that use rare commands may not work). Please submit a GitHub issue if you find something that doesn't work.*
-4. *Support for PSO Dreamcast Trial Edition and the December 2000 prototype is somewhat incomplete and probably never will be complete. These versions are rather unstable and seem to crash often, but it's not obvious whether it's because they're prototypes or because newserv sends data they can't handle.*
-5. *Creating a game works and battle setup behaves mostly normally, but starting a battle doesn't work.*
+1. *newserv's implementations of these versions are based on disassembly of the client executables and have never been tested.*
+2. *BB games are mostly playable, but there are still some unimplemented features (for example, some quests that use rare commands may not work). Please submit a GitHub issue if you find something that doesn't work.*
+3. *Creating a game works and battle setup behaves mostly normally, but starting a battle doesn't work.*
 
 ## Setup
 
@@ -113,8 +90,8 @@ There is a fairly recent macOS ARM64 release on the newserv GitHub repository. Y
 There is a fairly recent Windows release on the newserv GitHub repository also. It's built with Cygwin, and all the necessary DLL files should be included. That said, I've only tested it on my own machine and there is no CI for Windows builds like there is for macOS and Linux, so if it doesn't work for you, please open a GitHub issue to let me know.
 
 If you're not using a release from the GitHub repository, do this to build newserv:
-1. If you're on Windows, install Cygwin. While doing so, install the `cmake`, `gcc-core`, `gcc-g++`, `git`, `libevent2.1_7`, `make`, and `zlib` packages. Do the rest of these steps inside a Cygwin shell (not a Windows cmd shell or PowerShell).
-2. Make sure you have CMake and libevent installed. (On macOS, `brew install cmake libevent`; on most Linuxes, `sudo apt-get install cmake libevent-dev`; on Windows, you already did this in step 1.)
+1. If you're on Windows, install Cygwin. While doing so, install the `cmake`, `gcc-core`, `gcc-g++`, `git`, `libevent2.1_7`, `make`, `libiconv`, and `zlib` packages. Do the rest of these steps inside a Cygwin shell (not a Windows cmd shell or PowerShell).
+2. Make sure you have CMake, libevent, and libiconv installed. (On macOS, `brew install cmake libevent libiconv`; on most Linuxes, `sudo apt-get install cmake libevent-dev`; on Windows, you already did this in step 1.)
 3. Build and install phosg (https://github.com/fuzziqersoftware/phosg).
 4. Optionally, install resource_dasm (https://github.com/fuzziqersoftware/resource_dasm). This will enable newserv to send memory patches and load DOL files on PSO GC clients. PSO GC clients can play PSO normally on newserv without this.
 5. Run `cmake . && make` in the newserv directory.
@@ -130,17 +107,23 @@ To use newserv in other ways (e.g. for translating data), see the end of this do
 
 ### Installing quests
 
-newserv automatically finds quests in the system/quests/ directory. To install your own quests, or to use quests you've saved using the proxy's "save files" option, just put them in that directory and name them appropriately.
+newserv automatically finds quests in the subdirectories of the system/quests/ directory. To install your own quests, or to use quests you've saved using the proxy's Save Files option, just put them in one of the subdirectories there and name them appropriately. The subdirectories and their behaviors (e.g. in which game modes they should appear and for which PSO versions) is defined in the QuestCategories field in config.json.
 
-Standard quest files should be named like `q###-CATEGORY-VERSION.EXT`, battle quests should be named like `b###-VERSION.EXT`, challenge quests should be named like `c###-VERSION.EXT` for Episode 1 or `d###-VERSION.EXT` for Episode 2, and Episode 3 download quests should be named like `e###-gc3.EXT`. The fields in each filename are:
+Within the category directories, quest files should be named like `q###-VERSION-LANGUAGE.EXT` (although the `q` is ignored, and can be any letter). The fields in each filename are:
 - `###`: quest number (this doesn't really matter; it should just be unique across the PSO version)
-- `CATEGORY`: ret = Retrieval, ext = Extermination, evt = Events, shp = Shops, vr = VR, twr = Tower, gv1/gv2/gv4 = Government (BB only), dl = Download (these don't appear during online play), 1p = Solo (BB only)
-- `VERSION`: dn = Dreamcast NTE, d1 = Dreamcast v1, dc = Dreamcast v2, pc = PC, gcn = GameCube Trial Edition, gc = GameCube Episodes 1 & 2, gc3 = Episode 3, xb = Xbox, bb = Blue Burst
+- `VERSION`: dn = Dreamcast NTE, dp = Dreamcast 11/2000 prototype, d1 = Dreamcast v1, dc = Dreamcast v2, pc = PC, gcn = GameCube Trial Edition, gc = GameCube Episodes 1 & 2, gc3 = Episode 3 (see below), xb = Xbox, bb = Blue Burst
+- `LANGUAGE`: j = Japanese, e = English, g = German, f = French, s = Spanish
 - `EXT`: file extension (see table below)
 
-For example, the GameCube version of Lost HEAT SWORD is in two files named `q058-ret-gc.bin` and `q058-ret-gc.dat`. newserv knows these files are quests because they're in the system/quests/ directory, it knows they're for PSO GC because the filenames contain `-gc`, and it puts them in the Retrieval category because the filenames contain `-ret`.
+For .dat files, the `LANGUAGE` token may be omitted. If it's present, then that .dat file will only be used for that language of the quest; if omitted, then that .dat file will be used for all languages of the quest.
 
-The type identifiers (`b`, `c`, `d`, `e`, or `q`) and categories are configurable. See QuestCategories in config.example.json for more information on how to make new categories or edit the existing categories.
+Some quests (mostly battle and challenge mode quests) have additional JSON metadata files that describe how the server should handle them. These metadata files are generally named similarly to their .bin and .dat counterparts, except the `VERSION` token may also be omitted if the metadata applies to all versions of the quest on all PSO versions.
+
+Some quests may also include a .pvr file, which contains an image used in the quest. These files are named similarly to their .bin and .dat counterparts.
+
+For example, the GameCube version of Lost HEAT SWORD is in two files named `q058-gc-e.bin` and `q058-gc.dat`. newserv knows these files are quests because they're in the system/quests/ directory, it knows they're for PSO GC because the filenames contain `-gc`, it knows this is the English version of the quest because the .bin filename ends with `-e` (even though the .dat filename does not), and it puts them in the Retrieval category because the files are within the retrieval/ directory within system/quests/.
+
+The GameCube and Xbox quest formats are very similar, but newserv treats them as different. If you want to use the same quest file for GameCube and Xbox clients, you can make one a symbolic link to the other.
 
 There are multiple PSO quest formats out there; newserv supports all of them. It can also decode any known format to standard .bin/.dat format. Specifically:
 
@@ -156,6 +139,7 @@ There are multiple PSO quest formats out there; newserv supports all of them. It
 | GCI (with key)   | .bin.gci and .dat.gci | Yes        | decode-gci       |
 | GCI (no key)     | .bin.gci and .dat.gci | Decode (3) | decode-gci (3)   |
 | GCI (Ep3)        | .bin.gci or .mnm.gci  | Yes        | decode-gci       |
+| GCI (Ep3 Trial)  | .bin.gci or .mnm.gci  | Decode (3) | decode-gci (3)   |
 | DLQ              | .bin.dlq and .dat.dlq | Yes        | decode-dlq       |
 | DLQ (Ep3)        | .bin.dlq or .mnm.dlq  | Yes        | decode-dlq       |
 | QST (online)     | .qst                  | Yes        | decode-qst       |
@@ -165,9 +149,9 @@ There are multiple PSO quest formats out there; newserv supports all of them. It
 1. *This is the default format. You can convert these to uncompressed format by running `newserv decompress-prs FILENAME.bin FILENAME.bind` (and similarly for .dat -> .datd)*
 2. *Similar to (1), to compress an uncompressed quest file: `newserv compress-prs FILENAME.bind FILENAME.bin` (and likewise for .datd -> .dat)*
 3. *Use the decode action to convert these quests to .bin/.dat format before putting them into the server's quests directory. If you know the encryption seed (serial number), pass it in as a hex string with the `--seed=` option. If you don't know the encryption seed, newserv will find it for you, which will likely take a long time.*
-4. *Episode 3 online quests don't go in the system/quests directory; they instead go in the system/ep3/maps-free or system/ep3/maps-quest directories. If you want an Episode 3 quest to be available for both online play and for downloading, the file must exist in both system/quests and in one of the map directories in system/ep3.*
+4. *Episode 3 quests don't go in the system/quests directory. See the Episode 3 section below.*
 
-Episode 3 download quests consist only of a .bin file - there is no corresponding .dat file. Episode 3 download quest files may be named with the .mnm extension instead of .bin, since the format is the same as the standard map files (in system/ep3/). These files can be encoded in any of the formats described above, except .qst. There are no encrypted Episode 3 GCI formats because the game doesn't encrypt quests saved to the memory card, unlike Episodes 1&2.
+Episode 3 download quests consist only of a .bin file - there is no corresponding .dat file. Episode 3 download quest files may be named with the .mnm extension instead of .bin, since the format is the same as the standard map files (in system/ep3/). These files can be encoded in any of the formats described above, except .qst.
 
 When newserv indexes the quests during startup, it will warn (but not fail) if any quests are corrupt or in unrecognized formats.
 
@@ -177,29 +161,37 @@ All quests, including those originally in GCI or DLQ format, are treated as onli
 
 ### Episode 3 features
 
-The following Episode 3 features work well:
+newserv supports many features unique to Episode 3:
 * CARD battles. Not every combination of abilities has been tested yet, so if you find a feature or card ability that doesn't work like it's supposed to, please make a GitHub issue and describe the situation (the attacking card(s), defending card(s), and ability or condition that didn't work).
-* Tournaments. (But they don't work like Sega's tournaments did - see below)
+* Spectator teams.
+* Tournaments. (But they work differently than Sega's tournaments did - see below)
 * Downloading quests.
 * Trading cards.
 * Participating in card auctions. (The auction contents must be configured in config.json.)
+* Decorations in lobbies. Currently only images are supported; the game also supports loading custom 3D models in lobbies, but newserv does not implement this (yet).
 
-The following Episode 3 features are implemented, but are only partially tested:
-* Spectator teams. There is a known issue that prevents viewing battles unless you're in the spectator team when the battle begins, and spectating clients sometimes crash for an unknown reason.
-* Battle replays also sometimes cause the client to crash during the replay. Using the $playrec command is therefore not recommended.
+#### Battle records
 
-Tournaments work differently than they did on Sega's servers. Tournaments can be created with the `create-tournament` shell command, which enables players to register for them. (Use `help` to see all the arguments - there are many!) The `start-tournament` shell command starts the tournament (and prevents further registrations), but this doesn't schedule any matches. Instead, players who are ready to play their next match can all stand at the rightmost 4-player battle table in the same CARD lobby, and the tournament match will start automatically.
+After playing a battle, you can save the record of the battle with the $saverec command. You can then replay the battle later by using the $playrec command in a lobby - this will create a spectator team and play the recording of the battle as if it were happening in realtime. Note that there is a bug in older versions of Dolphin that seems to be frequently triggered when playing battle records, which causes the emulator to crash with the message `QObject::~QObject: Timers cannot be stopped from another thread`. To avoid this, use the latest version of Dolphin.
+
+#### Tournaments
+
+Tournaments work differently than they did on Sega's servers. Tournaments can be created with the `create-tournament` shell command, which enables players to register for them. (Use `help` to see all the arguments - there are many!) The `start-tournament` shell command starts the tournament (and prevents further registrations), but this doesn't schedule any matches. Instead, players who are ready to play their next match can all stand at the 4-player battle table near the lobby warp in the same CARD lobby, and the tournament match will start automatically.
 
 These tournament semantics mean that there can be multiple matches in the same tournament in play simultaneously, and not all matches in a round must be complete before the next round can begin - only the matches preceding each individual match must be complete for that match to be playable.
 
-Because newserv gives all players 1000000 meseta, there is no reward for winning a tournament. This may change in the future.
+The Meseta rewards for winning tournament matches can be configured in config.json.
+
+#### Episode 3 files
 
 Episode 3 state and game data is stored in the system/ep3 directory. The files in there are:
 * card-definitions.mnr: Compressed card definition list, sent to Episode 3 clients at connect time. Card stats and abilities can be changed by editing this file.
 * card-definitions.mnrd: Decompressed version of the above. If present, newserv will use this instead of the compressed version, since this is easier to edit.
 * card-text.mnr: Compressed card text archive. Generally only used for debugging.
+* card-text.mnrd: Decompressed card text archive; same format as TextCardE.bin. Generally only used for debugging.
 * com-decks.json: COM decks used in tournaments. The default decks in this file come from logs from Sega's servers, so the file doesn't include every COM deck Sega ever made - the rest are probably lost to time.
-* maps-free/ and maps-quest/: Online free battle and quest maps (.mnm/.bin/.mnmd/.bind files). Free battle and quest files have exactly the same format; the only difference between the files in these directories is which section of the menu they will appear in on the client.
+* maps/: Online free battle and quest maps (.mnm/.bin/.mnmd/.bind files). newserv comes with all the original online and offline maps, including Story Mode quests. If you don't want the offline maps and quests to be playable online, delete the .bind files system/ep3/maps.
+* maps-download/: Download maps and quests (.mnm/.bin/.mnmd/.bind files). There are two subcategories by default (download maps and Trial Edition download maps), but you can add more by editing QuestCategories in config.json. Categories that have flag 0x40 (Ep3 download) set are indexed from this directory; all others are indexed from system/quests/. Files in maps-download/ subdirectories have the same format as those in the maps/ directory, but should be named like `e###-gc3-LANGUAGE.EXT` (similar to how non-Episode 3 quests are named in the system/quests/ directory). If you want a map to be available for online play and for downloading, the file must exist in both maps/ and in a maps-download/ subdirectory (a symbolic link is acceptable).
 * tournament-state.json: State of all active tournaments. This file is automatically written when any tournament changes state for any reason (e.g. a tournament is created/started/deleted or a match is resolved).
 
 There is no public editor for Episode 3 maps and quests, but the format is described fairly thoroughly in src/Episode3/DataIndexes.hh (see the MapDefinition structure). You'll need to use `newserv decompress-prs ...` to decompress .bin or .mnm files before editing them, but you don't need to compress the files again to use them - just put the .bind or .mnmd file in the maps directory and newserv will make it available.
@@ -276,18 +268,22 @@ Some chat commands (see below) have the same basic function on the proxy server
 
 ### Chat commands
 
-The server's shell supports a variety of administration commands. If the interactive shell is enabled, you can enter these commands at any time, even if the prompt isn't visible. Run `help` in the server's shell to see all of the commands and how to use them.
-
-newserv also supports a variety of commands players can use via the chat interface. Any chat message that begins with `$` is treated as a chat command. (If you actually want to send a chat message starting with `$`, type `$$` instead.)
+newserv supports a variety of commands players can use by chatting in-game. Any chat message that begins with `$` is treated as a chat command. (If you actually want to send a chat message starting with `$`, type `$$` instead.)
 
 Some commands only work on the game server and not on the proxy server. The chat commands are:
 
 * Information commands
     * `$li`: Shows basic information about the lobby or game you're in. If you're on the proxy server, shows information about your connection instead (remote Guild Card number, client ID, etc.).
+    * `$ping` (game server only): Shows round-trip ping time from the server to you.
     * `$what` (game server only): Shows the type, name, and stats of the nearest item on the ground.
+    * `$matcount` (game server only): Shows how many of each type of material you've used.
 
 * Debugging commands
     * `$debug` (game server only): Enable or disable debug. You need the DEBUG permission in your user license to use this command. When debug is enabled, you'll see in-game messages from the server when you take certain actions. You'll also be placed into the highest available slot in lobbies and games instead of the lowest, which is useful for finding commands for which newserv doesn't handle client IDs properly. This setting also disables certain safeguards and allows you to do some things that might crash your client.
+    * `$quest <number>`: Load a quest by quest number. Can be used to load battle or challenge quests with only one player present.
+    * `$qcall <function-id>`: Call a quest function on your client.
+    * `$qset <flag-num>` or `$qclear <flag-num>`: Set or clear a global quest flag for everyone in the game.
+    * `$qsync <reg-num> <value>`: Set a quest register's value on your client. `<reg-num>` should be either rXX (e.g. r60) or fXX (e.g. f60); if the latter, `<value>` is parsed as a floating-point value instead of as an integer.
     * `$gc` (game server only): Send your own Guild Card to yourself.
     * `$persist` (game server only): Enable or disable persistence for the current lobby or game. This determines whether the lobby/game is deleted when the last player leaves. You need the DEBUG permission in your user license to use this command because there are no game state checks when you do this. For example, if you make a game persistent, start a quest, then leave the game, the game can't be joined by anyone but also can't be deleted.
     * `$sc <data>`: Send a command to yourself.
@@ -295,31 +291,42 @@ Some commands only work on the game server and not on the proxy server. The chat
 
 * Personal state commands
     * `$arrow <color-id>`: Changes your lobby arrow color.
-    * `$secid <section-id>`: Sets your override section ID. After running this command, any games you create will use your override section ID for rare drops instead of your character's actual section ID. To revert to your actual section id, run `$secid` with no name after it. On the proxy server, this will not work if the remote server controls item drops (e.g. on BB, or on Schtserv with server drops enabled).
-    * `$rand <seed>`: Sets your override random seed (specified as a 32-bit hex value). This will make any games you create use the given seed for rare enemies. This also makes item drops deterministic in Blue Burst games hosted by newserv. On the proxy server, this command can cause desyncs with other players in the same game, since they will not see the overridden random seed. To remove the override, run `$rand` with no arguments.
+    * `$secid <section-id>`: Sets your override section ID. After running this command, any games you create will use your override section ID for rare drops instead of your character's actual section ID. To revert to your actual section id, run `$secid` with no name after it. On the proxy server, this will not work if the remote server controls item drops (e.g. on BB, or on Schtserv with server drops enabled). If the server does not allow cheat mode anywhere (that is, "CheatModeBehavior" is "Off" in config.json), this command does nothing.
+    * `$rand <seed>`: Sets your override random seed (specified as a 32-bit hex value). This will make any games you create use the given seed for rare enemies. This also makes item drops deterministic in Blue Burst games hosted by newserv. On the proxy server, this command can cause desyncs with other players in the same game, since they will not see the overridden random seed. To remove the override, run `$rand` with no arguments. If the server does not allow cheat mode anywhere (that is, "CheatModeBehavior" is "Off" in config.json), this command does nothing.
+    * `$ln [name-or-type]`: Sets the lobby number. Visible only to you. This command exists because some non-lobby maps can be loaded as lobbies with invalid lobby numbers. See the "GC lobby types" and "Ep3 lobby types" entries in the information menu for acceptable values here. Note that non-lobby maps do not have a lobby counter, so there's no way to exit the lobby without using either `$ln` again or `$exit`. On the game server, `$ln` reloads the lobby immediately; on the proxy server, it doesn't take effect until you load another lobby yourself (which means you'll like have to use `$exit` to escape). Run this command with no argument to return to the default lobby.
     * `$exit`: If you're in a lobby, sends you to the main menu (which ends your proxy session, if you're in one). If you're in a game or spectator team, sends you to the lobby (but does not end your proxy session if you're in one). Does nothing if you're in a non-Episode 3 game and no quest is in progress.
     * `$patch <name>`: Run a patch on your client. `<name>` must exactly match the name of a patch on the server.
 
 * Blue Burst player commands (game server only)
     * `$bbchar <username> <password> <1-4>`: Use this command when playing on a non-BB version of PSO. If the username and password are correct, this command converts your current character to BB format and saves it on the server in the given slot. Any character already in that slot is overwritten.
-    * `$edit <stat> <value>`: Modifies your character data.
+    * `$edit <stat> <value>`: Modifies your character data. If the server does not allow cheat mode anywhere (that is, "CheatModeBehavior" is "Off" in config.json), this command does nothing.
+    * `$save`: Saves your character, system, and Guild Card data immediately. (By default, your character is saved every 60 seconds while online, and your account and Guild Card data are saved whenever they change.)
 
 * Game state commands (game server only)
     * `$maxlevel <level>`: Sets the maximum level for players to join the current game. (This only applies when joining; if a player joins and then levels up past this level during the game, they are not kicked out, but won't be able to rejoin if they leave.)
     * `$minlevel <level>`: Sets the minimum level for players to join the current game.
     * `$password <password>`: Sets the game's join password. To unlock the game, run `$password` with nothing after it.
-    * `$spec`: Toggles the allow spectators flag. If any players are spectating when this flag is disabled, they will be sent back to the lobby.
-    * `$saverec <name>`: Save the recording of the last Episode 3 battle.
-    * `$playrec <name>`: Play a battle recording. This command creates a spectator team and replays the specified battle log within it. There is a known issue which causes spectators to crash in some cases, so use of this command is currently not recommended.
+    * `$itemtable`: Switches between using the client's or the server's drop table. No effect on BB (the server's drop table is always used). The server's rare tables are defined in JSON files in the system/item-tables directory.
+    * `$drop`: Enables or disables all item drops from boxes and enemies in the current game.
+
+* Episode 3 commands (game server only)
+    * `$spec`: Toggles the allow spectators flag for Episode 3 games. If any players are spectating when this flag is disabled, they will be sent back to the lobby.
+    * `$inftime`: Toggles infinite-time mode. Must be used before starting a battle. If infinite-time mode is enabled, the overall and per-phase time limits will be disabled regardless of the values chosen during battle setup. After completing a battle, infinite-time mode is reset to the server's default value (which can be set in Episode3BehaviorFlags in config.json).
+    * `$defrange <min>-<max>`: Sets the DEF dice range for the next battle. If this is used, the dice range set during battle rules setup will apply only to ATK dice; DEF dice will use this range instead. Assist cards and other dice effects will still apply. Dice exchange also still applies if it is enabled.
+    * `$stat <what>`: Shows a statistic about your player or team in the current battle. `<what>` can be `duration`, `fcs-destroyed`, `cards-destroyed`, `damage-given`, `damage-taken`, `opp-cards-destroyed`, `own-cards-destroyed`, `move-distance`, `cards-set`, `fcs-set`, `attack-actions-set`, `techs-set`, `assists-set`, `defenses-self`, `defenses-ally`, `cards-drawn`, `max-attack-damage`, `max-combo`, `attacks-given`, `attacks-taken`, `sc-damage`, `damage-defended`, or `rank`.
+    * `$surrender`: Causes your team to immediately lose the current battle.
+    * `$saverec <name>`: Saves the recording of the last battle.
+    * `$playrec <name>`: Plays a battle recording. This command creates a spectator team and replays the specified battle log within it. There is a bug in Dolphin that makes use of this command unstable in emulation (see the "Battle records" section above).
 
 * Cheat mode commands
     * `$cheat`: Enables or disables cheat mode for the current game. All other cheat mode commands do nothing if cheat mode is disabled. By default, cheat mode is off in new games but can be enabled; there is an option in config.json that allows you to disable cheat mode entirely, or set it to on by default in new games.
     * `$infhp` / `$inftp`: Enables or disables infinite HP or TP mode. Applies to only you. In infinite HP mode, one-hit KO attacks will still kill you.
-    * `$warpme <area-id>`: Warps yourself to the given area.
-    * `$warpall <area-id>`: Warps everyone in the game to the given area. You must be the leader to use this command, unless you're on the proxy server.
-    * `$next`: Warps yourself to the next area.
+    * `$warpme <floor-id>`: Warps yourself to the given floor.
+    * `$warpall <floor-id>`: Warps everyone in the game to the given floor. You must be the leader to use this command, unless you're on the proxy server.
+    * `$next`: Warps yourself to the next floor.
     * `$swa`: Enables or disables switch assist. When enabled, the server will attempt to automatically unlock two-player doors in solo games if you step on both switches sequentially.
     * `$item <desc>` (or `$i <desc>`): Create an item. `desc` may be a description of the item (e.g. "Hell Saber +5 0/10/25/0/10") or a string of hex data specifying the item code. Item codes are 16 hex bytes; at least 2 bytes must be specified, and all unspecified bytes are zeroes. If you are on the proxy server, you must not be using Blue Burst for this command to work. On the game server, this command works for all versions.
+    * `$unset <index>`: In an Episode 3 battle, removes one of your set cards from the field. `<index>` is the index of the set card as it appears on your screen - 1 is the card next to your SC's icon, 2 is the card to the right of 1, etc. This does not cause a Hunters-side SC to lose HP, as they normally do when their items are destroyed.
 
 * Configuration commands
     * `$event <event>`: Sets the current holiday event in the current lobby. Holiday events are documented in the "Using $event" item in the information menu. If you're on the proxy server, this applies to all lobbies and games you join, but only you will see the new event - other players will not.
@@ -341,18 +348,25 @@ Some versions of PSO DC will connect to a private server if you just set their D
 
 If you're emulating PSO DC or have a disc image, you can patch the appropriate files within the disc image to make it connect to any address you want. Creating such a patch is also beyond the scope of this document.
 
-Finally, if you're emulating PSO DC, you can modify the loaded executable in memory to make it connect anywhere you want. There is a script included with newserv that can do this for Flycast. The script only works on macOS because it uses memwatch, which is specifically for macOS, but a similar technique could be done manually using scanmem on Linux or Cheat Engine on Windows. (The script is fairly short, and what it does should be easy to understand so you can duplicate its effects with scanmem or Cheat Engine.)
+#### PSO DC on Flycast
 
-To use the script, do this:
+If you're emulating PSO DC, all versions will connect to newserv by setting the following options in Flycast's `emu.cfg` file under `[network]`:
+- DNS = Your newserv's server address (newserv's DNS server must be running on port 53)
+- EmulateBBA = yes
+- Enable = yes
+
+It is also necessary to save any DNS information to the flash memory of the Dreamcast to use the BBA - the easiest way to do this is to use the website option in USv2 and then choose the save to flash option.
+
+Once set up, the EU and US versions will work without any extra set up (other than the HL Check Disable code for USv2), while the JP versions require HL Check Disable codes to be running.
+
+If the server is running on the same machine as Flycast, this might not work, even if you point Flycast's DNS queries at your local IP address (instead of 127.0.0.1). In this case, you can modify the loaded executable in memory to make it connect anywhere you want. There is a script included with newserv that can do this on macOS; a similar technique could be done manually using scanmem on Linux or Cheat Engine on Windows. To use the script, do this:
 1. Build and install memwatch (https://github.com/fuzziqersoftware/memwatch).
-2. Start Flycast and run PSO. (You must run the script below after PSO is loaded - it won't work if you run it before loading the game.)
+2. Start Flycast and run PSO. (You must start PSO before running the script; it won't work if you run the script before loading the game.)
 3. Run `sudo patch_flycast_memory.py <original-destination>`. Replace `<original-destination>` with the hostname that PSO wants to connect to (you can find this out by using Wireshark and looking for DNS queries). The script may take up to a minute; you can continue using Flycast while it runs, but don't start an online game until the script is done.
 4. Run newserv and start an online game in PSO.
 
 If you use this method, you'll have to run the script every time you start PSO in Flycast, but you won't have to run it again if you start another online game without restarting emulation.
 
-Finally, the script takes an optional second argument that allows you to redirect the connection elsewhere (instead of the local machine). This allows you to connect directly to remote servers if desired.
-
 #### PSO PC
 
 The version of PSO PC I have has the server addresses starting at offset 0x29CB34 in pso.exe. Using a hex editor, change those to "localhost" (without quotes) if you just want to connect to a locally-running newserv instance. Alternatively, you can add an entry to the Windows hosts file (C:\Windows\System32\drivers\etc\hosts) to redirect the connection to 127.0.0.1 (localhost) or any other IP address.
@@ -365,17 +379,14 @@ If you have PSO Plus or Episode III, it won't want to connect to a server on the
 
 #### PSO GC on Dolphin
 
-If you have BBA support via a tap interface or via the HLE/built-in interface, you may be able to just set the DNS server address (as you would on a real GameCube, above) and it may work.
+If you're using the HLE BBA type, set the BBA's DNS server address to newserv's IP address and it should work. (If newserv is on the same machine as Dolphin, try your local IP address or 127.0.0.1.) In PSO, use the example values below in PSO's network configuration.
 
-If you're using a version of Dolphin with tapserver support, you can make it connect to a newserv instance running on the same machine via the tapserver interface. You do not need to install or run tapserver, and this works for all PSO versions without any of the dual-interface trickery described above. To do this:
+If you're using the TAP BBA type, you'll have to set PSO's network settings appropriately for your tap interface. Set the DNS server address in PSO's network settings to newserv's IP address.
+
+If you're using a version of Dolphin with tapserver support, you can make it connect to a newserv instance running on the same machine via the tapserver interface. You do not need to install or run tapserver. To do this:
 1. Set Dolphin's BBA type to tapserver (Config -> GameCube -> SP1).
 2. Enable newserv's IP stack simulator according to the comments in config.json and start newserv.
-3. In PSO, you have to configure the network settings manually (DHCP doesn't work), but the actual values don't matter as long as they're valid IP addresses. Example values:
-  - IP address: `10.0.1.5`
-  - Subnet mask: `255.255.255.0`
-  - Default gateway: `10.0.1.1`
-  - DNS server address 1: `10.0.1.1`
-  - Leave everything else blank
+3. In PSO's network settings, enable DHCP ("Automatically obtain an IP address"), set DNS server address to "Automatic", and leave DHCP Hostname as "Not set". Leave the proxy server settings blank.
 4. Start an online game.
 
 ### Connecting external clients
@@ -397,9 +408,9 @@ newserv has many CLI options, which can be used to access functionality other th
 * Convert a PSO GC or Episode 3 snapshot file to a BMP image (`decode-gci-snapshot`)
 * Find the likely round1 or round2 seed for a corrupt save file (`salvage-gci`)
 * Run a brute-force search for a decryption seed (`find-decryption-seed`)
-* Decode Shift-JIS text to UTF-16 (`decode-sjis`)
 * Convert quests in .gci, .vms, .dlq, or .qst format to .bin/.dat format (`decode-gci`, `decode-vms`, `decode-dlq`, `decode-qst`)
 * Convert quests in .bin/.dat to .qst format (`encode-qst`)
+* Convert text archives (e.g. TextEnglish.pr2) to JSON and vice versa (`decode-text-archive`, `encode-text-archive`)
 * Disassemble quest scripts (`disassemble-quest-script`)
 * Format Episode 3 game data in a human-readable manner (`show-ep3-maps`, `show-ep3-cards`)
 * Convert item data to a human-readable description, or vice versa (`describe-item`, `encode-item`)

TODO.md

@@ -0,0 +1,41 @@
+## General
+
+- Find a way to silence audio in RunDOL.s
+- Encapsulate BB server-side random state and make replays deterministic
+- Implement choice search
+- Write a simple status API
+- Implement per-game logging
+- Build an exception-handling abstraction in ChatCommands that shows formatted error messages in all cases
+- Make reloading happen on separate threads so compression doesn't block active clients
+- Implement decrypt/encrypt actions for VMS files
+- Make UI strings localizable (e.g. entries in menus, welcome message, etc.)
+- Figure out what causes the corruption message on PC proxy sessions and fix it
+- Make $edit for DC/PC
+- Add an idle connection timeout for proxy sessions
+
+## Episode 3
+
+- Make disconnecting during a tournament match cause you to forfeit the match
+- Enforce tournament deck restrictions (e.g. rank checks, No Assist option) when populating COMs at tournament start time
+- It may be possible to send spectators back to the waiting room after a non-tournament battle by sending 6xB4x05 with environment 0x19, then 6xB4x3B again; try this
+- Add support for recording battles on the proxy server (both in primary and spectator teams)
+- When `reload ep3` happens and the defs file is changed, send the new defs file to all connected players who aren't in a game (if this even works - when exactly does the client decompress the defs file from the server?)
+- Make `reload licenses` not vulnerable to online players' licenses overwriting licenses on disk somehow
+- Implement ranks (based on total Meseta earned)
+
+## PSO XBOX
+
+- Fix receiving Guild Cards from non-Xbox players
+- Make the Guild Card description field in SavedPlayerDataBB longer to accommodate XB descriptions (0x200 bytes)
+
+## PSOBB
+
+- Find any remaining mismatches in enemy indexes / experience
+- Fix some edge cases on the BB proxy server (e.g. Change Ship)
+- Implement less-common subcommands
+    - 6xD8: Add S-rank weapon special
+- Test team commands
+    - Test all EA subcommands (a few are still not implemented)
+    - 6xC1, 6xC2, 6xCD, 6xCE: Team invites/administration (not implemented)
+    - Fix invite member menu
+- Implement story progress flags for unlocking quests

notes/AITalk.bin-format.txt

@@ -0,0 +1,16 @@
+struct AITalkBin {
+  be_uint32_t num_scs;
+  be_uint32_t sc_offsets[num_scs];
+
+  struct SCDialogueEntry {
+    be_uint32_t num_entries;
+    be_uint32_t unknown_a1;
+    be_uint32_t size; // in bytes
+    struct WhenEntry {
+      be_uint32_t when;
+      be_uint32_t percent_chance; // 0-100
+      be_uint32_t count;
+      be_uint32_t string_ids[count];
+    } __attribute__((packed));
+  } __attribute__((packed));
+} __attribute__((packed));

notes/ar-codes.txt

@@ -25,6 +25,18 @@
 (Ep3 USA) Auto-press A as fast as possible during loading screens
 042F9AC0 60000000
 
+(Ep3 USA) Replace loading screen A button sounds with random sounds
+042F9B18 4804BB19
+042F9B1C 5463063E
+042F9B20 60631400
+042F9B24 64630005
+042F9B28 38800000
+
+(Ep3 USA) Change color of loading screens
+(Replace AA, RR, GG, BB appropriately)
+042FA704 3CC0AARR
+042FA708 60C6GGBB
+
 (Ep3 USA) Use 16:9 aspect ratio
 04383DC8 4BC87F99
 0400BD60 C042DED0
@@ -49,7 +61,7 @@
 (Ep3 USA) Disable lobby event music (but keep the visuals)
 040B705C 38000000
 
-(Ep3 USA) Enable unused fourth Pinz's Shop choice
+(Ep3 USA) Enable Pinz's Shop Super Card Capsule Machine as a fourth option
 043101C0 38800004
 04310238 2C1D0004
 04487E8C 000000C8
@@ -172,5 +184,83 @@ TODO: Figure out more debug message conditionals (vars/functions) and add them h
 (Ep3 USA) Dressing room always accessible
 041A16FC 38600001
 
+(Ep3 USA) Full dressing room v1
+Can't change your class, but you start with your existing appearance
+Go online with this code on after using the dressing room to fully save changes
+0418EB5C 60000000
+042A0184 389D0370
+042A0188 387E2120
+
+(Ep3 USA) Full dressing room v2
+Can change your class, but you start with the default appearance
+Go online with this code on after using the dressing room to fully save changes
+04186ECC 4BFFFFD8
+042A0184 389D0370
+042A0188 387E2120
+
 (Ep3 USA) Replace Options menu with debug menu
 04149E70 38600019
+
+(Ep3 USA) Jukebox is free
+0430D1DC 48000024
+
+(Ep3 USA) Use own character in battle (online only)
+041FFAB0 4800001C
+042A54D8 4BD5B0F9
+04200A34 4BDFFB9D
+041FFA9C 4BE00B35
+040005D0 38600000
+040005D4 3CA08049
+040005D8 80A54160
+040005DC 2805000F
+040005E0 41820008
+040005E4 481E8E24
+040005E8 80ADA448
+040005EC 7C042800
+040005F0 41820008
+040005F4 481E8E14
+040005F8 38600001
+040005FC 4E800020
+
+(Ep3 USA) Disable chat smut filter
+0412F8B8 7D0802A6
+0412F8BC 7C661B78
+0412F8C0 7C872378
+0412F8C4 48217285
+0412F8C8 38A30001
+0412F8CC 7CE33B78
+0412F8D0 7CC43378
+0412F8D4 7D0803A6
+0412F8D8 4BEDEBF4
+
+(Ep3 USA) Metal tiles don't appear in Simulator map
+04296904 4E800020
+
+(Ep3 USA) Enable Boooo and Laughter soundchat sounds
+Note: Without a TextEnglish.pr2/pr3 patch, the menu items for these sounds will be blank (but they will still work)
+0430B734 38800029
+0430B770 2C1F0029
+0430B59C 2C030029
+0430B5A8 5460083C
+0430B5B4 7C63022E
+0442B690 80258026
+0442B694 8227852D
+0442B698 80308031
+0442B69C 8A3F8532
+0442B6A0 8A408533
+0442B6A4 8A418A28
+0442B6A8 8A388A29
+0442B6AC 8A39852E
+0442B6B0 802F853D
+0442B6B4 85348535
+0442B6B8 853B8536
+0442B6BC 8537852B
+0442B6C0 853A853C
+0442B6C4 853E8044
+0442B6C8 80458046
+0442B6CC 80478048
+0442B6D0 8049804A
+0442B6D4 804B804C
+0442B6D8 804D804E
+0442B6DC 804F802A
+0442B6E0 802C0000

notes/dc-proto-connection-addresses.txt

@@ -1,5 +1,5 @@
 DC NTE: pso02.dricas.ne.jp
 Nov 2000 proto: test1.st-pso.games.sega.net
-Dec 2000 proto: sg107634.csrd.sega.co.jp
+Dec 2000 proto: sg107634.csrd.sega.co.jp OR master.pso.dream-key.com
 Jan 2001 proto: master.pso.dream-key.com
 Aug 2001 proto (v2): ???

notes/decrypt-gci-tests.txt

@@ -1,4 +0,0 @@
-./newserv decrypt-gci-save --sys=8P-GPSE-PSO3_SYSTEM.gci 8P-GPSE-PSO3_CHARACTER.gci
-./newserv decrypt-gci-save --sys=8P-GPSE-PSO3_SYSTEM.gci 8P-GPSE-PSO3_GUILDCARD.gci
-./newserv decrypt-gci-save --sys=8P-GPOE-PSO_SYSTEM.gci 8P-GPOE-PSO_CHARACTER.gci
-./newserv decrypt-gci-save --sys=8P-GPOE-PSO_SYSTEM.gci 8P-GPOE-PSO_GUILDCARD.gci

notes/ep3-vip-card-notes.txt

@@ -1,29 +0,0 @@
-N1, N2, N3, N4 => use 8041F800 table
-R1, R2, R3, R4 => use 8041F8A0 table
-
-
-(Episode 3 USA) Able to find VIP cards offline (but still very rare)
-042C0B20 4800000C
-
-
-P(activate) is the probability that any transformation is attempted at all
-P(f/success) defines the probability range: so the actual probability is a
-  uniform random number between P(activate) and P(activate) * P(f/success)
-count  P(activate)  P(f/success)  P(vip)
-0-4    0.0          0.0           0.0
-5-10   0.01923077   0.55          0.005
-11-16  0.021276595  0.6           0.0045454544
-17-24  0.023809524  0.7           0.004347826
-25-32  0.027027028  0.7           0.004
-33-40  0.03125      0.8           0.0038461538
-41-52  0.037037037  0.8           0.0035714286
-53-99  0.05         0.9           0.0033333334
-
-0-4    0.0          0.0           0.0
-5-10   0.020408163  0.55          0.005
-11-16  0.022727273  0.6           0.004761905
-17-24  0.025641026  0.7           0.0045454544
-25-32  0.029411765  0.7           0.005
-33-40  0.03448276   0.7           0.005
-41-52  0.041666668  0.8           0.0045454544
-53-99  0.05263158   0.9           0.004347826

notes/psobb/jpbb-resources/help_city_ja.lst

@@ -0,0 +1,3 @@
+.\data\help2-0-ja.png
+.\data\help0-4-ja.png
+.\data\help0-5-ja.png

notes/psobb/jpbb-resources/help_lobby_ja.lst

@@ -0,0 +1,6 @@
+.\data\help2-0-ja.png
+.\data\help0-0-ja.png
+.\data\help0-1-ja.png
+.\data\help0-2-ja.png
+.\data\help0-3-ja.png
+.\data\help0-7-ja.png

notes/psobb/jpbb-resources/help_prompt_city_ja.lst

@@ -0,0 +1 @@
+.\data\help1-1-ja.png

notes/psobb/jpbb-resources/help_prompt_lobby_ja.lst

@@ -0,0 +1 @@
+.\data\help1-0-ja.png

notes/psobb/psobb-notes.txt

@@ -0,0 +1,39 @@
+PSOBB SUPPORT FILES, NOTES & RESOURCES
+
+--------------------------------------------------------------------------------
+CLIENT LOCALIZATION
+
+By default PSOBB loads everything in Japanese so it requires some extra files
+to properly implement the English localization from SOA, these files are offered
+here inside the usbb-resources folder for your convenience they are the same ones
+from the old official USBB client
+
+To use them, you just need to drag and drop all its contents into your client's 
+data folder. Then if the client's internal lang flag is set correctly to English 
+will load all the correct texts from these files.
+
+In case you want to play in Japanese, just use the default Tethealla client and 
+delete all the files including _e or _eng in the names and then the game will 
+default everything to its original Japanese language.
+
+Just in case, there's the jpbb-resources folder with the latest localization 
+changes made on the official JPBB for an extra backup.
+
+---------------------------------------------------------------------------------
+PSOBB EP1,2,4 ORIGINAL VANILLA DROP TABLES/RATES
+
+Included in the vanilla-tables folder I placed the original files I created for the
+Schtserv vanilla for backup purposes as they are already implemented into the main
+newserv logic. 
+
+These tables will offer you the experience as close as possible to the original SEGA
+servers for PSOBB JP up to the latest patch before the servers shutdown, so besides a 
+fully functional Episode IV experience, the tables also include the latest special items
+which where added to some Episode 1 and Episode 2 in Ultimate for certain section ID's
+
+Vanilla Tables and rates are the same ones as the Schtserv Wiki for reference:
+https://bbwiki.schtserv.com/index.php/Drops-ep1
+https://bbwiki.schtserv.com/index.php/Drops-ep2
+https://bbwiki.schtserv.com/index.php/Drops-ep4
+
+

notes/psobb/usbb-resources/help_city_en.lst

@@ -0,0 +1,3 @@
+.\data\help2-0-en.png
+.\data\help0-4-en.png
+.\data\help0-5-en.png

notes/psobb/usbb-resources/help_lobby_en.lst

@@ -0,0 +1,6 @@
+.\data\help2-0-en.png
+.\data\help0-0-en.png
+.\data\help0-1-en.png
+.\data\help0-2-en.png
+.\data\help0-3-en.png
+.\data\help0-7-en.png

notes/psobb/usbb-resources/help_prompt_city_en.lst

@@ -0,0 +1 @@
+.\data\help1-1-en.png

notes/psobb/usbb-resources/help_prompt_lobby_en.lst

@@ -0,0 +1 @@
+.\data\help1-0-en.png

src/AFSArchive.cc

@@ -0,0 +1,86 @@
+#include "AFSArchive.hh"
+
+#include <stdio.h>
+#include <string.h>
+
+#include <phosg/Encoding.hh>
+#include <phosg/Filesystem.hh>
+#include <phosg/Strings.hh>
+
+using namespace std;
+
+AFSArchive::AFSArchive(shared_ptr<const string> data)
+    : data(data) {
+  struct FileHeader {
+    be_uint32_t magic;
+    le_uint32_t num_files;
+  } __attribute__((packed));
+
+  struct FileEntry {
+    le_uint32_t offset;
+    le_uint32_t size;
+  } __attribute__((packed));
+
+  StringReader r(*this->data);
+  const auto& header = r.get<FileHeader>();
+  if (header.magic != 0x41465300) { // 'AFS\0'
+    throw runtime_error("file is not an AFS archive");
+  }
+
+  while (this->entries.size() < header.num_files) {
+    const auto& entry = r.get<FileEntry>();
+    this->entries.emplace_back(Entry{.offset = entry.offset, .size = entry.size});
+  }
+}
+
+pair<const void*, size_t> AFSArchive::get(size_t index) const {
+  const auto& entry = this->entries.at(index);
+  if (entry.offset > this->data->size()) {
+    throw out_of_range("entry begins beyond end of archive");
+  }
+  if (entry.offset + entry.size > this->data->size()) {
+    throw out_of_range("entry extends beyond end of archive");
+  }
+
+  return make_pair(this->data->data() + entry.offset, entry.size);
+}
+
+string AFSArchive::get_copy(size_t index) const {
+  auto ret = this->get(index);
+  return string(reinterpret_cast<const char*>(ret.first), ret.second);
+}
+
+StringReader AFSArchive::get_reader(size_t index) const {
+  auto ret = this->get(index);
+  return StringReader(ret.first, ret.second);
+}
+
+string AFSArchive::generate(const vector<string>& files, bool big_endian) {
+  return big_endian ? AFSArchive::generate_t<true>(files) : AFSArchive::generate_t<false>(files);
+}
+
+template <bool IsBigEndian>
+string AFSArchive::generate_t(const vector<string>& files) {
+  using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
+
+  StringWriter w;
+  w.put_u32b(0x41465300); // 'AFS\0'
+  w.put<U32T>(files.size());
+
+  // It seems entries are aligned to 0x800-byte boundaries, and the file's
+  // header is always 0x80000 (!) bytes, most of which is unused
+  uint32_t data_offset = 0x80000;
+  for (const auto& file : files) {
+    w.put<U32T>(data_offset);
+    w.put<U32T>(file.size());
+    data_offset = (data_offset + file.size() + 0x7FF) & (~0x7FF);
+  }
+
+  w.extend_to(0x80000);
+  for (const auto& file : files) {
+    w.write(file);
+    w.extend_to((w.size() + 0x7FF) & (~0x7FF));
+  }
+
+  return std::move(w.str());
+}

src/AFSArchive.hh

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <stdint.h>
+
+#include <memory>
+#include <phosg/Filesystem.hh>
+#include <phosg/Strings.hh>
+#include <string>
+#include <unordered_map>
+
+class AFSArchive {
+public:
+  AFSArchive(std::shared_ptr<const std::string> data);
+  ~AFSArchive() = default;
+
+  struct Entry {
+    uint64_t offset;
+    uint32_t size;
+  };
+  inline const std::vector<Entry>& all_entries() const {
+    return this->entries;
+  }
+
+  std::pair<const void*, size_t> get(size_t index) const;
+  std::string get_copy(size_t index) const;
+  StringReader get_reader(size_t index) const;
+
+  static std::string generate(const std::vector<std::string>& files, bool big_endian);
+
+private:
+  template <bool IsBigEndian>
+  static std::string generate_t(const std::vector<std::string>& files);
+
+  std::shared_ptr<const std::string> data;
+  std::vector<Entry> entries;
+};

src/ARCodeTranslator-Stub.hh

@@ -0,0 +1,8 @@
+#pragma once
+
+#include <stdexcept>
+#include <string>
+
+inline void run_ar_code_translator(const std::string&, const std::string&, const std::string&) {
+  throw std::runtime_error("resource_file is not available; install it and rebuild newserv");
+}

src/ARCodeTranslator.cc

@@ -0,0 +1,154 @@
+#include "ARCodeTranslator.hh"
+
+#include <phosg/Filesystem.hh>
+#include <phosg/Strings.hh>
+#include <resource_file/ExecutableFormats/DOLFile.hh>
+
+using namespace std;
+
+void run_ar_code_translator(const std::string& initial_directory, const std::string& use_file, const std::string& command) {
+  string directory = initial_directory;
+  while (ends_with(directory, "/")) {
+    directory.resize(directory.size() - 1);
+  }
+  PrefixedLogger log("[ar-trans] ");
+
+  unordered_map<string, shared_ptr<DOLFile>> files;
+  for (const auto& filename : list_directory(directory)) {
+    if (ends_with(filename, ".dol")) {
+      string name = filename.substr(0, filename.size() - 4);
+      string path = directory + "/" + filename;
+      files.emplace(name, new DOLFile(path.c_str()));
+      log.info("Loaded %s", name.c_str());
+    }
+  }
+
+  string source_filename;
+  shared_ptr<DOLFile> source_file;
+  auto find_match = [&](std::shared_ptr<DOLFile> target_file, uint32_t source_address) -> uint32_t {
+    const DOLFile::Section* source_section = nullptr;
+    for (const auto& sec : source_file->sections) {
+      if (source_address >= sec.address && source_address < sec.address + sec.data.size()) {
+        source_section = &sec;
+        break;
+      }
+    }
+    if (!source_section) {
+      throw runtime_error("source address not within any section");
+    }
+    size_t source_offset = source_address - source_section->address;
+    size_t source_bytes_available_after = source_section->data.size() - source_offset;
+    log.info("(find_match) Source offset = %08zX with %08zX bytes available after", source_offset, source_bytes_available_after);
+
+    for (size_t match_length = 4;
+         match_length < min<size_t>(source_bytes_available_after, 0x100);
+         match_length += 4) {
+      size_t num_matches = 0;
+      size_t last_match_address = 0;
+      StringReader source_r(source_section->data.data() + source_offset, match_length);
+      for (const auto& target_section : target_file->sections) {
+        for (size_t target_section_offset = 0;
+             target_section_offset + match_length <= target_section.data.size();
+             target_section_offset += 4) {
+          source_r.go(0);
+          StringReader target_r(target_section.data.data() + target_section_offset, match_length);
+          size_t z;
+          for (z = 0; z < match_length; z += 4) {
+            if (source_section->is_text) {
+              uint32_t source_opcode = source_r.get_u32b();
+              uint32_t target_opcode = target_r.get_u32b();
+              uint32_t source_class = source_opcode & 0xFC000000;
+              if (source_class != (target_opcode & 0xFC000000)) {
+                break;
+              }
+              if (source_class == 0x48000000) {
+                source_opcode &= 0xFC000003;
+                target_opcode &= 0xFC000003;
+              } else if (source_class == 0x40000000) {
+                source_opcode &= 0xFFFF0003;
+                target_opcode &= 0xFFFF0003;
+              }
+              if (source_opcode != target_opcode) {
+                break;
+              }
+            } else {
+              if (source_r.get_u32l() != target_r.get_u32l()) {
+                break;
+              }
+            }
+          }
+          if (z == match_length) {
+            num_matches++;
+            last_match_address = target_section.address + target_section_offset;
+          }
+        }
+      }
+      log.info("(find_match) For match length %zX, %zu matches found", match_length, num_matches);
+      if (num_matches == 1) {
+        return last_match_address;
+      } else if (num_matches == 0) {
+        throw runtime_error("did not find exactly one match");
+      }
+    }
+    throw runtime_error("scan field too long; too many matches");
+  };
+
+  auto handle_command = [&](const string& command) -> void {
+    auto tokens = split(command, ' ');
+    if (tokens.empty()) {
+      throw runtime_error("no command given");
+    }
+    strip_trailing_whitespace(tokens[tokens.size() - 1]);
+
+    if (tokens[0] == "use") {
+      source_filename = tokens.at(1);
+      source_file = files.at(source_filename);
+    } else if (tokens[0] == "match") {
+      if (!source_file) {
+        throw runtime_error("no source file selected");
+      }
+
+      uint32_t source_addr = stoul(tokens.at(1), nullptr, 16);
+      for (const auto& it : files) {
+        if (it.second == source_file) {
+          log.info("(%s) %08" PRIX32, it.first.c_str(), source_addr);
+        } else {
+          try {
+            uint32_t match_addr = find_match(it.second, source_addr);
+            log.info("(%s) %08" PRIX32, it.first.c_str(), match_addr);
+          } catch (const exception& e) {
+            log.error("(%s) failed: %s", it.first.c_str(), e.what());
+          }
+        }
+      }
+    } else if (!tokens[0].empty()) {
+      throw runtime_error("unknown command");
+    }
+  };
+
+  if (!use_file.empty()) {
+    source_filename = use_file;
+    source_file = files.at(source_filename);
+  }
+
+  if (!command.empty()) {
+    handle_command(command);
+  } else {
+    while (!feof(stdin)) {
+      if (!source_filename.empty()) {
+        fprintf(stdout, "ar-trans:%s/%s> ", directory.c_str(), source_filename.c_str());
+      } else {
+        fprintf(stdout, "ar-trans:%s> ", directory.c_str());
+      }
+      fflush(stdout);
+
+      string command = fgets(stdin);
+      try {
+        handle_command(command);
+      } catch (const exception& e) {
+        log.error("Failed: %s", e.what());
+      }
+    }
+    fputc('\n', stdout);
+  }
+}

src/ARCodeTranslator.hh

@@ -0,0 +1,5 @@
+#pragma once
+
+#include <string>
+
+void run_ar_code_translator(const std::string& initial_directory, const std::string& use_file, const std::string& command);

src/BMLArchive.cc

@@ -21,7 +21,7 @@ template <bool IsBigEndian>
 struct BMLHeaderEntry {
   using U32T = typename std::conditional<IsBigEndian, be_uint32_t, le_uint32_t>::type;
 
-  ptext<char, 0x20> filename;
+  pstring<TextEncoding::ASCII, 0x20> filename;
   U32T compressed_size;
   parray<uint8_t, 0x04> unknown_a1;
   U32T decompressed_size;
@@ -52,7 +52,7 @@ void BMLArchive::load_t() {
     size_t gvm_offset = offset;
     offset = (offset + entry.compressed_gvm_size + 0x1F) & (~0x1F);
 
-    this->entries.emplace(entry.filename, Entry{data_offset, entry.compressed_size, gvm_offset, entry.compressed_gvm_size});
+    this->entries.emplace(entry.filename.decode(), Entry{data_offset, entry.compressed_size, gvm_offset, entry.compressed_gvm_size});
   }
 }