compress DOL files before sending them to clients

src/Compression.cc

@@ -216,8 +216,9 @@ string prs_compress(
 
 class ControlStreamReader {
 public:
-  ControlStreamReader(StringReader& r) : r(r),
-                                         bits(0x0000) {}
+  ControlStreamReader(StringReader& r)
+      : r(r),
+        bits(0x0000) {}
 
   bool read() {
     if (!(this->bits & 0x0100)) {

src/FunctionCompiler.cc

@@ -4,6 +4,7 @@
 #include <string.h>
 
 #include <phosg/Filesystem.hh>
+#include <phosg/Time.hh>
 #include <stdexcept>
 
 #ifdef HAVE_RESOURCE_FILE
@@ -11,6 +12,7 @@
 #endif
 
 #include "CommandFormats.hh"
+#include "Compression.hh"
 #include "Loggers.hh"
 
 using namespace std;
@@ -247,7 +249,8 @@ bool FunctionCodeIndex::patch_menu_empty(uint32_t specific_version) const {
   return true;
 }
 
-DOLFileIndex::DOLFileIndex(const string& directory) {
+DOLFileIndex::DOLFileIndex(const string& directory, bool compress)
+    : files_compressed(compress) {
   if (!function_compiler_available()) {
     function_compiler_log.info("Function compiler is not available");
     return;
@@ -274,16 +277,63 @@ DOLFileIndex::DOLFileIndex(const string& directory) {
       dol->name = name;
 
       string path = directory + "/" + filename;
-      dol->data = load_file(path);
+      string file_data = load_file(path);
+
+      string description;
+      if (this->files_compressed) {
+        uint64_t start = now();
+        string compressed = prs_compress(file_data);
+        StringWriter w;
+        if (compressed.size() >= file_data.size()) {
+          w.put_u32b(0);
+          w.put_u32b(file_data.size());
+          w.write(file_data);
+        } else {
+          w.put_u32b(compressed.size());
+          w.put_u32b(file_data.size());
+          w.write(compressed);
+        }
+        while (w.size() & 3) {
+          w.put_u8(0);
+        }
+        dol->data = std::move(w.str());
+        uint64_t diff = now() - start;
+
+        string orig_size_str = format_size(file_data.size());
+        string compressed_size_str = format_size(dol->data.size());
+        string time_str = format_duration(diff);
+
+        if (compressed.size() >= file_data.size()) {
+          function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s) (inefficient compression; using uncompressed version)",
+              dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str());
+          description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str());
+        } else {
+          function_compiler_log.info("Loaded and compressed DOL file %s (%s -> %s, %s)",
+              dol->name.c_str(), orig_size_str.c_str(), compressed_size_str.c_str(), time_str.c_str());
+          description = string_printf("$C6%s$C7\n%s\n%s (orig)", dol->name.c_str(), compressed_size_str.c_str(), orig_size_str.c_str());
+        }
+
+      } else {
+        StringWriter w;
+        w.put_u32b(0);
+        w.put_u32b(file_data.size());
+        w.write(file_data);
+        while (w.size() & 3) {
+          w.put_u8(0);
+        }
+        dol->data = std::move(w.str());
+
+        string orig_size_str = format_size(dol->data.size());
+        function_compiler_log.info("Loaded DOL file %s (%s)", filename.c_str(), orig_size_str.c_str());
+
+        description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), orig_size_str.c_str());
+      }
 
       this->name_to_file.emplace(dol->name, dol);
       this->item_id_to_file.emplace_back(dol);
 
-      string size_str = format_size(dol->data.size());
-      string description = string_printf("$C6%s$C7\n%s", dol->name.c_str(), size_str.c_str());
       menu->items.emplace_back(dol->menu_item_id, decode_sjis(dol->name),
           decode_sjis(description), MenuItem::Flag::REQUIRES_SEND_FUNCTION_CALL);
-      function_compiler_log.info("Loaded DOL file %s", filename.c_str());
 
     } catch (const exception& e) {
       function_compiler_log.warning("Failed to load DOL file %s: %s", filename.c_str(), e.what());

src/FunctionCompiler.hh

@@ -76,12 +76,13 @@ struct DOLFileIndex {
     std::string data;
   };
 
+  bool files_compressed;
   std::vector<std::shared_ptr<DOLFile>> item_id_to_file;
   std::map<std::string, std::shared_ptr<DOLFile>> name_to_file;
   std::shared_ptr<const Menu> menu;
 
   DOLFileIndex() = default;
-  explicit DOLFileIndex(const std::string& directory);
+  DOLFileIndex(const std::string& directory, bool compress);
 
   inline bool empty() const {
     return this->name_to_file.empty() && this->item_id_to_file.empty();

src/ServerState.cc

@@ -24,6 +24,7 @@ ServerState::ServerState(const char* config_filename, bool is_replay)
       allow_saving(true),
       item_tracking_enabled(true),
       episode_3_send_function_call_enabled(false),
+      enable_dol_compression(false),
       catch_handler_exceptions(true),
       ep3_behavior_flags(0),
       run_shell_behavior(RunShellBehavior::DEFAULT),
@@ -654,6 +655,12 @@ void ServerState::parse_config(shared_ptr<const JSONObject> config_json) {
     this->episode_3_send_function_call_enabled = false;
   }
 
+  try {
+    this->enable_dol_compression = d.at("CompressDOLFiles")->as_bool();
+  } catch (const out_of_range&) {
+    this->enable_dol_compression = false;
+  }
+
   try {
     this->catch_handler_exceptions = d.at("CatchHandlerExceptions")->as_bool();
   } catch (const out_of_range&) {
@@ -860,5 +867,5 @@ void ServerState::compile_functions() {
 
 void ServerState::load_dol_files() {
   config_log.info("Loading DOL files");
-  this->dol_file_index.reset(new DOLFileIndex("system/dol"));
+  this->dol_file_index.reset(new DOLFileIndex("system/dol", this->enable_dol_compression));
 }

src/ServerState.hh

@@ -55,6 +55,7 @@ struct ServerState {
   bool allow_saving;
   bool item_tracking_enabled;
   bool episode_3_send_function_call_enabled;
+  bool enable_dol_compression;
   bool catch_handler_exceptions;
   uint32_t ep3_behavior_flags;
   RunShellBehavior run_shell_behavior;