incentive/psopeeps-newserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compress DOL files before sending them to clients

Browse Source
This commit is contained in:
Martin Michelsen
2023-05-27 09:57:36 -07:00
parent 9ec72212cf
commit 54a734e049
8 changed files with 200 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Compression.cc
+3 -2
View File
@@ -216,8 +216,9 @@ string prs_compress(
class ControlStreamReader {
public:
ControlStreamReader(StringReader& r) : r(r),
bits(0x0000) {}
ControlStreamReader(StringReader& r)
: r(r),
bits(0x0000) {}
bool read() {
if (!(this->bits & 0x0100)) {
src/FunctionCompiler.cc
+55 -5
View File
@@ -4,6 +4,7 @@
#include <string.h>
#include <phosg/Filesystem.hh>
#include <phosg/Time.hh>
#include <stdexcept>
#ifdef HAVE_RESOURCE_FILE
@@ -11,6 +12,7 @@
#endif
#include "CommandFormats.hh"
#include "Compression.hh"
#include "Loggers.hh"
using namespace std;
@@ -247,7 +249,8 @@ bool FunctionCodeIndex::patch_menu_empty(uint32_t specific_version) const {
return true;
}
DOLFileIndex::DOLFileIndex(const string& directory) {
DOLFileIndex::DOLFileIndex(const string& directory, bool compress)
: files_compressed(compress) {
if (!function_compiler_available()) {
function_compiler_log.info("Function compiler is not available");
return;
@@ -274,16 +277,63 @@ DOLFileIndex::DOLFileIndex(const string& directory) {
dol->name = name;
string path = directory + "/" + filename;
dol->data = load_file(path);
string file_data = load_file(path);
string description;
if (this->files_compressed) {
uint64_t start = now();
string compressed = prs_compress(file_data);
StringWriter w;
if (compressed.size() >= file_data.size()) {
w.put_u32b(0);
w.put_u32b(file_data.size());
w.write(file_data);
} else {
w.put_u32b(compressed.size());
w.put_u32b(file_data.size());
w.write(compressed);
}
while (w.size() & 3) {
w.put_u8(0);
}
dol->data = std::move(w.str());
uint64_t diff = now() - start;
string orig_size_str = format_size(file_data.size());
string compressed_size_str = format_size(dol->data.size());
string time_str = format_duration(diff);
if (compressed.size() >= file_data.size()) {
function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s) (inefficient compression; using uncompressed version)",
dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str());
description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str());
} else {
function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s)",
dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str());
description = string_printf("$C6%s$C7\n%s\n%s (orig)", dol->name.c_str(), compressed_size_str.c_str(), orig_size_str.c_str());
}
} else {
StringWriter w;
w.put_u32b(0);
w.put_u32b(file_data.size());
w.write(file_data);
while (w.size() & 3) {
w.put_u8(0);
}
dol->data = std::move(w.str());
string orig_size_str = format_size(dol->data.size());
function_compiler_log.info("Loaded DOL file %s (%s)", filename.c_str(), orig_size_str.c_str());
description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str());
}
this->name_to_file.emplace(dol->name, dol);
this->item_id_to_file.emplace_back(dol);
string size_str = format_size(dol->data.size());
string description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), size_str.c_str());
menu->items.emplace_back(dol->menu_item_id, decode_sjis(dol->name),
decode_sjis(description), MenuItem::Flag::REQUIRES_SEND_FUNCTION_CALL);
function_compiler_log.info("Loaded DOL file %s", filename.c_str());
} catch (const exception& e) {
function_compiler_log.warning("Failed to load DOL file %s: %s", filename.c_str(), e.what());
src/FunctionCompiler.hh
+2 -1
View File
@@ -76,12 +76,13 @@ struct DOLFileIndex {
std::string data;
};
bool files_compressed;
std::vector<std::shared_ptr<DOLFile>> item_id_to_file;
std::map<std::string, std::shared_ptr<DOLFile>> name_to_file;
std::shared_ptr<const Menu> menu;
DOLFileIndex() = default;
explicit DOLFileIndex(const std::string& directory);
DOLFileIndex(const std::string& directory, bool compress);
inline bool empty() const {
return this->name_to_file.empty() && this->item_id_to_file.empty();
src/ServerState.cc
+8 -1
View File
@@ -24,6 +24,7 @@ ServerState::ServerState(const char* config_filename, bool is_replay)
allow_saving(true),
item_tracking_enabled(true),
episode_3_send_function_call_enabled(false),
enable_dol_compression(false),
catch_handler_exceptions(true),
ep3_behavior_flags(0),
run_shell_behavior(RunShellBehavior::DEFAULT),
@@ -654,6 +655,12 @@ void ServerState::parse_config(shared_ptr<const JSONObject> config_json) {
this->episode_3_send_function_call_enabled = false;
}
try {
this->enable_dol_compression = d.at("CompressDOLFiles")->as_bool();
} catch (const out_of_range&) {
this->enable_dol_compression = false;
}
try {
this->catch_handler_exceptions = d.at("CatchHandlerExceptions")->as_bool();
} catch (const out_of_range&) {
@@ -860,5 +867,5 @@ void ServerState::compile_functions() {
void ServerState::load_dol_files() {
config_log.info("Loading DOL files");
this->dol_file_index.reset(new DOLFileIndex("system/dol"));
this->dol_file_index.reset(new DOLFileIndex("system/dol", this->enable_dol_compression));
}
src/ServerState.hh
+1
View File
@@ -55,6 +55,7 @@ struct ServerState {
bool allow_saving;
bool item_tracking_enabled;
bool episode_3_send_function_call_enabled;
bool enable_dol_compression;
bool catch_handler_exceptions;
uint32_t ep3_behavior_flags;
RunShellBehavior run_shell_behavior;
system/config.example.json
+5
View File
@@ -309,6 +309,11 @@
// enable Episode 3 patches by default; it only does so if this option is on.
// "EnableEpisode3SendFunctionCall": true,
// If this setting is enabled, newserv will compress DOL files at startup
// time. This makes startup time slower but makes clients' loading time much
// faster.
"CompressDOLFiles": true,
// By default, the server keeps track of items in all games, even for versions
// other than Blue Burst. This enables use of the $what command, as well as
// protection against item duplication cheats (the cheater is disconnected
system/ppc/PRSDecompress.inc.s
+103
View File
@@ -0,0 +1,103 @@
prs_decompress__start:
# r3 = dest ptr (used as write ptr)
subi r3, r3, 1
# r4 = src ptr (used as read ptr)
subi r4, r4, 1
# r5 = dest size (converted to ptr to last valid output byte)
add r5, r5, r3
# r6 = src size (converted to ptr to last valid input byte)
add r6, r6, r4
# r7 = control bits + guard bits
li r7, 0
# r8 = temp for offset/count
# r9 = original dest ptr - 1 (used for computing return value)
mr r9, r3
# r10 = temp for reading/writing data
# r11 = second-level saved LR, temp for offset/count
# r12 = saved LR
mflr r12
prs_decompress__next_opcode:
bl prs_decompress__cmp_control_bit_and_return_in_r10
beq prs_decompress__control_0
prs_decompress__control_1:
bl prs_decompress__read_byte_to_r10
bl prs_decompress__write_byte_from_r10
b prs_decompress__next_opcode
prs_decompress__control_0:
bl prs_decompress__cmp_control_bit_and_return_in_r10
beq prs_decompress__control_00
prs_decompress__control_01:
bl prs_decompress__read_byte_to_r10
rlwinm r8, r10, 29, 27, 31 # low 5 bits of offset
rlwinm r11, r10, 0, 29, 31 # size
addi r11, r11, 2
bl prs_decompress__read_byte_to_r10
rlwimi. r8, r10, 5, 19, 26 # high 8 bits of offset
bne prs_decompress__control_01_not_stop_opcode
sub r3, r3, r9
mtlr r12
blr
prs_decompress__control_01_not_stop_opcode:
ori r8, r8, 0xE000
cmplwi r11, 2
bne prs_decompress__control_01_not_extended_count
bl prs_decompress__read_byte_to_r10
addi r11, r10, 1
prs_decompress__control_01_not_extended_count:
mtctr r11
b prs_decompress__control_00_01_copy
prs_decompress__control_00:
bl prs_decompress__cmp_control_bit_and_return_in_r10
rlwinm r11, r10, 1, 30, 30
bl prs_decompress__cmp_control_bit_and_return_in_r10
rlwimi r11, r10, 0, 31, 31
addi r11, r11, 2
mtctr r11
bl prs_decompress__read_byte_to_r10
ori r8, r10, 0xFF00
prs_decompress__control_00_01_copy:
# r8 = src offset (negative 16-bit value)
# ctr = byte count to copy
oris r8, r8, 0xFFFF
add r8, r8, r3 # r8 = copy src ptr (minus 1, for lbzu)
prs_decompress__control_00_01_copy_again:
lbzu r10, [r8 + 1]
bl prs_decompress__write_byte_from_r10
bdnz prs_decompress__control_00_01_copy_again
b prs_decompress__next_opcode
prs_decompress__cmp_control_bit_and_return_in_r10:
andi. r10, r7, 0x0100
bne prs_decompress__skip_read
mflr r8
bl prs_decompress__read_byte_to_r10
mtlr r8
mr r7, r10
ori r7, r7, 0xFF00
prs_decompress__skip_read:
andi. r10, r7, 1
rlwinm r7, r7, 31, 17, 31
blr
prs_decompress__read_byte_to_r10:
cmp r4, r6
bge prs_decompress__return_error
lbzu r10, [r4 + 1]
blr
prs_decompress__write_byte_from_r10:
cmp r3, r5
bge prs_decompress__return_error
stbu [r3 + 1], r10
blr
prs_decompress__return_error:
li r3, -1
mtlr r12
blr
system/ppc/RunDOL.s
+23 -1
View File
@@ -53,8 +53,25 @@ copy_code_to_low_memory__again:
run_dol:
lwz r30, [r31 + 0x10] # r30 = DOL base ptr
lwz r30, [r31 + 0x10] # r30 = data base ptr
# Decompress the file first. If the compressed size is zero, then skip this
# step (the file is not compressed). The header consists of two fields:
# compressed size followed by decompressed size.
lwz r6, [r30]
cmplwi r6, 0
beq run_dol__not_compressed
lwz r5, [r30 + 4]
addi r4, r30, 8 # Compressed data immediately follows the 2 header fields
sub r3, r30, r5 # Decompress to immediately before the compressed data
mr r30, r3 # Save DOL header pointer for after decompression
bl prs_decompress
b run_dol__decompressed
run_dol__not_compressed:
addi r30, r30, 8
run_dol__decompressed:
# DOL files are very simple: they have up to 7 text sections, up to 11 data
# sections, and a BSS section and an entrypoint. No imports or other fancy
# things to do - we just have to move a bunch of bytes around.
@@ -119,6 +136,11 @@ flush_cached_code_writes:
prs_decompress:
.include PRSDecompress
return_end_ptr:
mflr r3
bctr