make DOLFileIndex support both compressed and uncompressed files simultaneously

src/FunctionCompiler.cc

@@ -257,8 +257,7 @@ bool FunctionCodeIndex::patch_menu_empty(uint32_t specific_version) const {
   return true;
 }
 
-DOLFileIndex::DOLFileIndex(const string& directory, bool compress)
+DOLFileIndex::DOLFileIndex(const string& directory) {
-    : files_compressed(compress) {
   if (!function_compiler_available()) {
     function_compiler_log.info("Function compiler is not available");
     return;
@@ -274,10 +273,12 @@ DOLFileIndex::DOLFileIndex(const string& directory, bool compress)
 
   uint32_t next_menu_item_id = 0;
   for (const auto& filename : list_directory_sorted(directory)) {
-    if (!ends_with(filename, ".dol")) {
+    bool is_dol = ends_with(filename, ".dol");
+    bool is_compressed_dol = ends_with(filename, ".dol.prs");
+    if (!is_dol && !is_compressed_dol) {
       continue;
     }
-    string name = filename.substr(0, filename.size() - 4);
+    string name = filename.substr(0, filename.size() - (is_compressed_dol ? 8 : 4));
 
     try {
       shared_ptr<DOLFile> dol(new DOLFile());
@@ -288,38 +289,24 @@ DOLFileIndex::DOLFileIndex(const string& directory, bool compress)
       string file_data = load_file(path);
 
       string description;
-      if (this->files_compressed) {
+      if (is_compressed_dol) {
-        uint64_t start = now();
+        size_t decompressed_size = prs_decompress_size(file_data);
-        string compressed = prs_compress(file_data);
+
         StringWriter w;
-        if (compressed.size() >= file_data.size()) {
-          w.put_u32b(0);
         w.put_u32b(file_data.size());
+        w.put_u32b(decompressed_size);
         w.write(file_data);
-        } else {
-          w.put_u32b(compressed.size());
-          w.put_u32b(file_data.size());
-          w.write(compressed);
-        }
         while (w.size() & 3) {
           w.put_u8(0);
         }
         dol->data = std::move(w.str());
-        uint64_t diff = now() - start;
 
-        string orig_size_str = format_size(file_data.size());
+        string compressed_size_str = format_size(file_data.size());
-        string compressed_size_str = format_size(dol->data.size());
+        string decompressed_size_str = format_size(decompressed_size);
-        string time_str = format_duration(diff);
+        function_compiler_log.info("Loaded compressed DOL file %s (%s -> %s)",
-
+            dol->name.c_str(), compressed_size_str.c_str(), decompressed_size_str.c_str());
-        if (compressed.size() >= file_data.size()) {
+        description = string_printf("$C6%s$C7\n%s\n%s (orig)",
-          function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s) (inefficient compression; using uncompressed version)",
+            dol->name.c_str(), compressed_size_str.c_str(), decompressed_size_str.c_str());
-              dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str());
-          description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str());
-        } else {
-          function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s)",
-              dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str());
-          description = string_printf("$C6%s$C7\n%s\n%s (orig)", dol->name.c_str(), compressed_size_str.c_str(), orig_size_str.c_str());
-        }
 
       } else {
         StringWriter w;
@@ -331,10 +318,9 @@ DOLFileIndex::DOLFileIndex(const string& directory, bool compress)
         }
         dol->data = std::move(w.str());
 
-        string orig_size_str = format_size(dol->data.size());
+        string size_str = format_size(dol->data.size());
-        function_compiler_log.info("Loaded DOL file %s (%s)", filename.c_str(), orig_size_str.c_str());
+        function_compiler_log.info("Loaded DOL file %s (%s)", filename.c_str(), size_str.c_str());
-
+        description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), size_str.c_str());
-        description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str());
       }
 
       this->name_to_file.emplace(dol->name, dol);

src/FunctionCompiler.hh

@@ -74,15 +74,15 @@ struct DOLFileIndex {
     uint32_t menu_item_id;
     std::string name;
     std::string data;
+    bool is_compressed;
   };
 
-  bool files_compressed;
   std::vector<std::shared_ptr<DOLFile>> item_id_to_file;
   std::unordered_map<std::string, std::shared_ptr<DOLFile>> name_to_file;
   std::shared_ptr<const Menu> menu;
 
   DOLFileIndex() = default;
-  DOLFileIndex(const std::string& directory, bool compress);
+  explicit DOLFileIndex(const std::string& directory);
 
   inline bool empty() const {
     return this->name_to_file.empty() && this->item_id_to_file.empty();

src/ServerState.cc

@@ -25,7 +25,6 @@ ServerState::ServerState(const char* config_filename, bool is_replay)
       item_tracking_enabled(true),
       drops_enabled(true),
       episode_3_send_function_call_enabled(false),
-      enable_dol_compression(false),
       catch_handler_exceptions(true),
       ep3_behavior_flags(0),
       run_shell_behavior(RunShellBehavior::DEFAULT),
@@ -663,12 +662,6 @@ void ServerState::parse_config(shared_ptr<const JSONObject> config_json) {
     this->episode_3_send_function_call_enabled = false;
   }
 
-  try {
-    this->enable_dol_compression = d.at("CompressDOLFiles")->as_bool();
-  } catch (const out_of_range&) {
-    this->enable_dol_compression = false;
-  }
-
   try {
     this->catch_handler_exceptions = d.at("CatchHandlerExceptions")->as_bool();
   } catch (const out_of_range&) {
@@ -921,5 +914,5 @@ void ServerState::compile_functions() {
 
 void ServerState::load_dol_files() {
   config_log.info("Loading DOL files");
-  this->dol_file_index.reset(new DOLFileIndex("system/dol", this->enable_dol_compression));
+  this->dol_file_index.reset(new DOLFileIndex("system/dol"));
 }

src/ServerState.hh

@@ -60,7 +60,6 @@ struct ServerState {
   bool item_tracking_enabled;
   bool drops_enabled;
   bool episode_3_send_function_call_enabled;
-  bool enable_dol_compression;
   bool catch_handler_exceptions;
   uint32_t ep3_behavior_flags;
   RunShellBehavior run_shell_behavior;

system/config.example.json

@@ -368,11 +368,6 @@
   // enable Episode 3 patches by default; it only does so if this option is on.
   // "EnableEpisode3SendFunctionCall": true,
 
-  // If this setting is enabled, newserv will compress DOL files at startup
-  // time. This makes startup time slower but makes clients' loading time much
-  // faster.
-  "CompressDOLFiles": false,
-
   // By default, the server keeps track of items in all games, even for versions
   // other than Blue Burst. This enables use of the $what command, as well as
   // protection against item duplication cheats (the cheater is disconnected