From d93866146a609041d1f46183a6b0e52f6c7d5386 Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Sat, 26 Aug 2023 21:00:01 -0700 Subject: [PATCH] make DOLFileIndex support both compressed and uncompressed files simultaneously --- src/FunctionCompiler.cc | 54 ++++++++++++++------------------------ src/FunctionCompiler.hh | 4 +-- src/ServerState.cc | 9 +------ src/ServerState.hh | 1 - system/config.example.json | 5 ---- 5 files changed, 23 insertions(+), 50 deletions(-) diff --git a/src/FunctionCompiler.cc b/src/FunctionCompiler.cc index 1ed82c1e..c3eefacc 100644 --- a/src/FunctionCompiler.cc +++ b/src/FunctionCompiler.cc @@ -257,8 +257,7 @@ bool FunctionCodeIndex::patch_menu_empty(uint32_t specific_version) const { return true; } -DOLFileIndex::DOLFileIndex(const string& directory, bool compress) - : files_compressed(compress) { +DOLFileIndex::DOLFileIndex(const string& directory) { if (!function_compiler_available()) { function_compiler_log.info("Function compiler is not available"); return; @@ -274,10 +273,12 @@ DOLFileIndex::DOLFileIndex(const string& directory, bool compress) uint32_t next_menu_item_id = 0; for (const auto& filename : list_directory_sorted(directory)) { - if (!ends_with(filename, ".dol")) { + bool is_dol = ends_with(filename, ".dol"); + bool is_compressed_dol = ends_with(filename, ".dol.prs"); + if (!is_dol && !is_compressed_dol) { continue; } - string name = filename.substr(0, filename.size() - 4); + string name = filename.substr(0, filename.size() - (is_compressed_dol ? 8 : 4)); try { shared_ptr dol(new DOLFile()); @@ -288,38 +289,24 @@ DOLFileIndex::DOLFileIndex(const string& directory, bool compress) string file_data = load_file(path); string description; - if (this->files_compressed) { - uint64_t start = now(); - string compressed = prs_compress(file_data); + if (is_compressed_dol) { + size_t decompressed_size = prs_decompress_size(file_data); + StringWriter w; - if (compressed.size() >= file_data.size()) { - w.put_u32b(0); - w.put_u32b(file_data.size()); - w.write(file_data); - } else { - w.put_u32b(compressed.size()); - w.put_u32b(file_data.size()); - w.write(compressed); - } + w.put_u32b(file_data.size()); + w.put_u32b(decompressed_size); + w.write(file_data); while (w.size() & 3) { w.put_u8(0); } dol->data = std::move(w.str()); - uint64_t diff = now() - start; - string orig_size_str = format_size(file_data.size()); - string compressed_size_str = format_size(dol->data.size()); - string time_str = format_duration(diff); - - if (compressed.size() >= file_data.size()) { - function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s) (inefficient compression; using uncompressed version)", - dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str()); - description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str()); - } else { - function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s)", - dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str()); - description = string_printf("$C6%s$C7\n%s\n%s (orig)", dol->name.c_str(), compressed_size_str.c_str(), orig_size_str.c_str()); - } + string compressed_size_str = format_size(file_data.size()); + string decompressed_size_str = format_size(decompressed_size); + function_compiler_log.info("Loaded compressed DOL file %s (%s -> %s)", + dol->name.c_str(), compressed_size_str.c_str(), decompressed_size_str.c_str()); + description = string_printf("$C6%s$C7\n%s\n%s (orig)", + dol->name.c_str(), compressed_size_str.c_str(), decompressed_size_str.c_str()); } else { StringWriter w; @@ -331,10 +318,9 @@ DOLFileIndex::DOLFileIndex(const string& directory, bool compress) } dol->data = std::move(w.str()); - string orig_size_str = format_size(dol->data.size()); - function_compiler_log.info("Loaded DOL file %s (%s)", filename.c_str(), orig_size_str.c_str()); - - description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str()); + string size_str = format_size(dol->data.size()); + function_compiler_log.info("Loaded DOL file %s (%s)", filename.c_str(), size_str.c_str()); + description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), size_str.c_str()); } this->name_to_file.emplace(dol->name, dol); diff --git a/src/FunctionCompiler.hh b/src/FunctionCompiler.hh index 22b34af1..6b0982c4 100644 --- a/src/FunctionCompiler.hh +++ b/src/FunctionCompiler.hh @@ -74,15 +74,15 @@ struct DOLFileIndex { uint32_t menu_item_id; std::string name; std::string data; + bool is_compressed; }; - bool files_compressed; std::vector> item_id_to_file; std::unordered_map> name_to_file; std::shared_ptr menu; DOLFileIndex() = default; - DOLFileIndex(const std::string& directory, bool compress); + explicit DOLFileIndex(const std::string& directory); inline bool empty() const { return this->name_to_file.empty() && this->item_id_to_file.empty(); diff --git a/src/ServerState.cc b/src/ServerState.cc index 500d1afc..09563b4a 100644 --- a/src/ServerState.cc +++ b/src/ServerState.cc @@ -25,7 +25,6 @@ ServerState::ServerState(const char* config_filename, bool is_replay) item_tracking_enabled(true), drops_enabled(true), episode_3_send_function_call_enabled(false), - enable_dol_compression(false), catch_handler_exceptions(true), ep3_behavior_flags(0), run_shell_behavior(RunShellBehavior::DEFAULT), @@ -663,12 +662,6 @@ void ServerState::parse_config(shared_ptr config_json) { this->episode_3_send_function_call_enabled = false; } - try { - this->enable_dol_compression = d.at("CompressDOLFiles")->as_bool(); - } catch (const out_of_range&) { - this->enable_dol_compression = false; - } - try { this->catch_handler_exceptions = d.at("CatchHandlerExceptions")->as_bool(); } catch (const out_of_range&) { @@ -921,5 +914,5 @@ void ServerState::compile_functions() { void ServerState::load_dol_files() { config_log.info("Loading DOL files"); - this->dol_file_index.reset(new DOLFileIndex("system/dol", this->enable_dol_compression)); + this->dol_file_index.reset(new DOLFileIndex("system/dol")); } diff --git a/src/ServerState.hh b/src/ServerState.hh index 45b1e664..5aab95ac 100644 --- a/src/ServerState.hh +++ b/src/ServerState.hh @@ -60,7 +60,6 @@ struct ServerState { bool item_tracking_enabled; bool drops_enabled; bool episode_3_send_function_call_enabled; - bool enable_dol_compression; bool catch_handler_exceptions; uint32_t ep3_behavior_flags; RunShellBehavior run_shell_behavior; diff --git a/system/config.example.json b/system/config.example.json index c62a5a2c..db3453fe 100644 --- a/system/config.example.json +++ b/system/config.example.json @@ -368,11 +368,6 @@ // enable Episode 3 patches by default; it only does so if this option is on. // "EnableEpisode3SendFunctionCall": true, - // If this setting is enabled, newserv will compress DOL files at startup - // time. This makes startup time slower but makes clients' loading time much - // faster. - "CompressDOLFiles": false, - // By default, the server keeps track of items in all games, even for versions // other than Blue Burst. This enables use of the $what command, as well as // protection against item duplication cheats (the cheater is disconnected