unify compression progress_fns

src/CommandFormats.hh

@@ -891,7 +891,7 @@ struct S_GoodLuckResult_BB_24 {
 // 25 (S->C): Gallon's Plan result (BB)
 // Sent in response to a 6xE1 command from the client.
 
-struct S_Unknown_BB_25 {
+struct S_GallonPlanResult_BB_25 {
   le_uint16_t unknown_a1 = 0;
   uint8_t offset1 = 0;
   uint8_t offset2 = 0;
@@ -4033,14 +4033,14 @@ struct G_AttackFinished_6x46 {
 struct G_CastTechnique_6x47 {
   G_ClientIDHeader header;
   uint8_t technique_number;
-  uint8_t unused;
+  uint8_t unused; // Must not be negative
   // Note: The level here isn't the actual tech level that was cast, if the
-  // level is > 15. In that case, a 6x8D is sent first, which contains the
+  // actual level is > 15. In that case, a 6x8D is sent first, which contains
-  // additional level which is added to this level at cast time. They probably
+  // the additional level which is added to this level at cast time. They
-  // did this for legacy reasons when dealing with v1/v2 compatibility, and
+  // probably did this for legacy reasons when dealing with v1/v2
-  // never cleaned it up.
+  // compatibility, and never cleaned it up.
   uint8_t level;
-  uint8_t target_count;
+  uint8_t target_count; // Must be in [0, 10]
   struct TargetEntry {
     le_uint16_t client_id;
     le_uint16_t unknown_a2;
@@ -4687,7 +4687,7 @@ struct G_Unknown_6x8A {
 
 // 6x8D: Set technique level override
 // This command is sent immediately before 6x47 if the technique level is above
-// 15. Presumably this was done for some backward-compatibility reason.
+// 15. Presumably this was done for compatibility between v1 and v2.
 
 struct G_SetTechniqueLevelOverride_6x8D {
   G_ClientIDHeader header;

src/Compression.cc

@@ -14,35 +14,15 @@
 using namespace std;
 
 template <>
-const char* name_for_enum<PRSCompressOptimalPhase>(PRSCompressOptimalPhase v) {
+const char* name_for_enum<CompressPhase>(CompressPhase v) {
   switch (v) {
-    case PRSCompressOptimalPhase::INDEX_SHORT_COPIES:
+    case CompressPhase::INDEX:
-      return "INDEX_SHORT_COPIES";
-    case PRSCompressOptimalPhase::INDEX_LONG_COPIES:
-      return "INDEX_LONG_COPIES";
-    case PRSCompressOptimalPhase::INDEX_EXTENDED_COPIES:
-      return "INDEX_EXTENDED_COPIES";
-    case PRSCompressOptimalPhase::CONSTRUCT_PATHS:
-      return "CONSTRUCT_PATHS";
-    case PRSCompressOptimalPhase::BACKTRACE_OPTIMAL_PATH:
-      return "BACKTRACE_OPTIMAL_PATH";
-    case PRSCompressOptimalPhase::GENERATE_RESULT:
-      return "GENERATE_RESULT";
-    default:
-      return "__UNKNOWN__";
-  }
-}
-
-template <>
-const char* name_for_enum<BC0CompressOptimalPhase>(BC0CompressOptimalPhase v) {
-  switch (v) {
-    case BC0CompressOptimalPhase::INDEX:
       return "INDEX";
-    case BC0CompressOptimalPhase::CONSTRUCT_PATHS:
+    case CompressPhase::CONSTRUCT_PATHS:
       return "CONSTRUCT_PATHS";
-    case BC0CompressOptimalPhase::BACKTRACE_OPTIMAL_PATH:
+    case CompressPhase::BACKTRACE_OPTIMAL_PATH:
       return "BACKTRACE_OPTIMAL_PATH";
-    case BC0CompressOptimalPhase::GENERATE_RESULT:
+    case CompressPhase::GENERATE_RESULT:
       return "GENERATE_RESULT";
     default:
       return "__UNKNOWN__";
@@ -245,19 +225,23 @@ struct PRSPathNode {
 };
 
 string prs_compress_optimal(
-    const void* in_data_v, size_t in_size, function<void(PRSCompressOptimalPhase, size_t, size_t)> progress_fn) {
+    const void* in_data_v, size_t in_size, ProgressCallback progress_fn) {
   const uint8_t* in_data = reinterpret_cast<const uint8_t*>(in_data_v);
 
   vector<PRSPathNode> nodes;
   nodes.resize(in_size + 1);
   nodes[0].bits_used = 18; // Stop command: 2 control bits and 2 data bytes
 
+  size_t copy_progress_max = 3 * in_size;
+  atomic<size_t> copy_progress;
+
   // Populate all possible short copies
-  {
+  std::thread short_window_thread([&]() -> void {
     WindowIndex<0x100, 5, true> window(in_data_v, in_size);
     while (window.offset < in_size) {
-      if ((window.offset & 0xFFF) == 0) {
+      if ((window.offset & 0xFFF) == 0 && progress_fn) {
-        progress_fn(PRSCompressOptimalPhase::INDEX_SHORT_COPIES, window.offset, 0);
+        size_t progress = copy_progress.fetch_add(0x1000) + 0x1000;
+        progress_fn(CompressPhase::INDEX, progress, copy_progress_max, 0);
       }
       auto& node = nodes[window.offset];
       auto match = window.get_best_match();
@@ -267,14 +251,15 @@ string prs_compress_optimal(
       }
       window.advance();
     }
-  }
+  });
 
   // Populate all possible long copies
-  {
+  std::thread long_window_thread([&]() -> void {
     WindowIndex<0x1FFF, 9, true> window(in_data_v, in_size);
     while (window.offset < in_size) {
-      if ((window.offset & 0xFFF) == 0) {
+      if ((window.offset & 0xFFF) == 0 && progress_fn) {
-        progress_fn(PRSCompressOptimalPhase::INDEX_LONG_COPIES, window.offset, 0);
+        size_t progress = copy_progress.fetch_add(0x1000) + 0x1000;
+        progress_fn(CompressPhase::INDEX, progress, copy_progress_max, 0);
       }
       auto& node = nodes[window.offset];
       auto match = window.get_best_match();
@@ -284,14 +269,15 @@ string prs_compress_optimal(
       }
       window.advance();
     }
-  }
+  });
 
   // Populate all possible extended copies
-  {
+  std::thread extended_window_thread([&]() -> void {
     WindowIndex<0x1FFF, 0x100, true> window(in_data_v, in_size);
     while (window.offset < in_size) {
-      if ((window.offset & 0xFFF) == 0) {
+      if ((window.offset & 0xFFF) == 0 && progress_fn) {
-        progress_fn(PRSCompressOptimalPhase::INDEX_EXTENDED_COPIES, window.offset, 0);
+        size_t progress = copy_progress.fetch_add(0x1000) + 0x1000;
+        progress_fn(CompressPhase::INDEX, progress, copy_progress_max, 0);
       }
       auto& node = nodes[window.offset];
       auto match = window.get_best_match();
@@ -301,13 +287,17 @@ string prs_compress_optimal(
       }
       window.advance();
     }
-  }
+  });
+
+  short_window_thread.join();
+  long_window_thread.join();
+  extended_window_thread.join();
 
   // For each node, populate the literal value, and the best ways to get to the
   // following nodes
   for (size_t z = 0; z < in_size; z++) {
     if ((z & 0xFFF) == 0) {
-      progress_fn(PRSCompressOptimalPhase::CONSTRUCT_PATHS, z, 0);
+      progress_fn(CompressPhase::CONSTRUCT_PATHS, z, in_size, 0);
     }
 
     auto& node = nodes[z];
@@ -362,7 +352,9 @@ string prs_compress_optimal(
   for (size_t z = in_size; z > 0;) {
     if ((z & ~0xFFF) != (last_progress_fn_call & ~0xFFF)) {
       last_progress_fn_call = z;
-      progress_fn(PRSCompressOptimalPhase::BACKTRACE_OPTIMAL_PATH, z, 0);
+      if (progress_fn) {
+        progress_fn(CompressPhase::BACKTRACE_OPTIMAL_PATH, z, in_size, 0);
+      }
     }
     size_t from_offset = nodes[z].from_offset;
     nodes[from_offset].to_offset = z;
@@ -375,7 +367,9 @@ string prs_compress_optimal(
   for (size_t offset = 0; offset < in_size;) {
     if ((offset & ~0xFFF) != (last_progress_fn_call & ~0xFFF)) {
       last_progress_fn_call = offset;
-      progress_fn(PRSCompressOptimalPhase::GENERATE_RESULT, offset, w.size());
+      if (progress_fn) {
+        progress_fn(CompressPhase::GENERATE_RESULT, offset, in_size, w.size());
+      }
     }
 
     const auto& node = nodes[offset];
@@ -449,7 +443,7 @@ string prs_compress_optimal(
 }
 
 PRSCompressor::PRSCompressor(
-    ssize_t compression_level, function<void(size_t, size_t)> progress_fn)
+    ssize_t compression_level, ProgressCallback progress_fn)
     : compression_level(compression_level),
       progress_fn(progress_fn),
       closed(false),
@@ -566,7 +560,7 @@ void PRSCompressor::move_forward_data_to_reverse_log(size_t size) {
   for (; size > 0; size--) {
     this->reverse_log.push_back(this->forward_log.at(this->reverse_log.end_offset()));
     if (this->progress_fn && ((this->reverse_log.end_offset() & 0xFFF) == 0)) {
-      this->progress_fn(this->reverse_log.end_offset(), this->output.size());
+      this->progress_fn(CompressPhase::GENERATE_RESULT, this->reverse_log.end_offset(), this->input_bytes, this->output.size());
     }
   }
 }
@@ -656,7 +650,7 @@ string prs_compress(
     const void* vdata,
     size_t size,
     ssize_t compression_level,
-    function<void(size_t, size_t)> progress_fn) {
+    ProgressCallback progress_fn) {
   PRSCompressor prs(compression_level, progress_fn);
   prs.add(vdata, size);
   return std::move(prs.close());
@@ -665,12 +659,12 @@ string prs_compress(
 string prs_compress(
     const string& data,
     ssize_t compression_level,
-    function<void(size_t, size_t)> progress_fn) {
+    ProgressCallback progress_fn) {
   return prs_compress(data.data(), data.size(), compression_level, progress_fn);
 }
 
 string prs_compress_indexed(
-    const void* in_data_v, size_t in_size, function<void(size_t, size_t)> progress_fn) {
+    const void* in_data_v, size_t in_size, ProgressCallback progress_fn) {
   const uint8_t* in_data = reinterpret_cast<const uint8_t*>(in_data_v);
 
   LZSSInterleavedWriter w;
@@ -680,7 +674,7 @@ string prs_compress_indexed(
   while (window.offset < in_size) {
     if (progress_fn && ((last_progress_fn_call_offset & ~0xFFF) != (window.offset & ~0xFFF))) {
       last_progress_fn_call_offset = window.offset;
-      progress_fn(window.offset, w.size());
+      progress_fn(CompressPhase::GENERATE_RESULT, window.offset, in_size, w.size());
     }
 
     auto match = window.get_best_match();
@@ -763,7 +757,7 @@ string prs_compress_indexed(
   return std::move(w.close());
 }
 
-string prs_compress_indexed(const string& data, function<void(size_t, size_t)> progress_fn) {
+string prs_compress_indexed(const string& data, ProgressCallback progress_fn) {
   return prs_compress_indexed(data.data(), data.size(), progress_fn);
 }
 
@@ -999,7 +993,7 @@ struct BC0PathNode {
 };
 
 string bc0_compress_optimal(
-    const void* in_data_v, size_t in_size, function<void(BC0CompressOptimalPhase, size_t, size_t)> progress_fn) {
+    const void* in_data_v, size_t in_size, ProgressCallback progress_fn) {
   const uint8_t* in_data = reinterpret_cast<const uint8_t*>(in_data_v);
 
   vector<BC0PathNode> nodes;
@@ -1010,8 +1004,8 @@ string bc0_compress_optimal(
   {
     WindowIndex<0x1000, 0x12> window(in_data_v, in_size);
     while (window.offset < in_size) {
-      if ((window.offset & 0xFFF) == 0) {
+      if ((window.offset & 0xFFF) == 0 && progress_fn) {
-        progress_fn(BC0CompressOptimalPhase::INDEX, window.offset, 0);
+        progress_fn(CompressPhase::INDEX, window.offset, in_size, 0);
       }
       auto& node = nodes[window.offset];
       auto match = window.get_best_match();
@@ -1026,8 +1020,8 @@ string bc0_compress_optimal(
   // For each node, populate the literal value, and the best ways to get to the
   // following nodes
   for (size_t z = 0; z < in_size; z++) {
-    if ((z & 0xFFF) == 0) {
+    if ((z & 0xFFF) == 0 && progress_fn) {
-      progress_fn(BC0CompressOptimalPhase::CONSTRUCT_PATHS, z, 0);
+      progress_fn(CompressPhase::CONSTRUCT_PATHS, z, in_size, 0);
     }
 
     auto& node = nodes[z];
@@ -1058,7 +1052,9 @@ string bc0_compress_optimal(
   for (size_t z = in_size; z > 0;) {
     if ((z & ~0xFFF) != (last_progress_fn_call & ~0xFFF)) {
       last_progress_fn_call = z;
-      progress_fn(BC0CompressOptimalPhase::BACKTRACE_OPTIMAL_PATH, z, 0);
+      if (progress_fn) {
+        progress_fn(CompressPhase::BACKTRACE_OPTIMAL_PATH, z, in_size, 0);
+      }
     }
     size_t from_offset = nodes[z].from_offset;
     nodes[from_offset].to_offset = z;
@@ -1071,7 +1067,9 @@ string bc0_compress_optimal(
   for (size_t offset = 0; offset < in_size;) {
     if ((offset & ~0xFFF) != (last_progress_fn_call & ~0xFFF)) {
       last_progress_fn_call = offset;
-      progress_fn(BC0CompressOptimalPhase::GENERATE_RESULT, offset, w.size());
+      if (progress_fn) {
+        progress_fn(CompressPhase::GENERATE_RESULT, offset, in_size, w.size());
+      }
     }
 
     const auto& node = nodes[offset];
@@ -1092,11 +1090,11 @@ string bc0_compress_optimal(
   return std::move(w.close());
 }
 
-string bc0_compress(const string& data, function<void(size_t, size_t)> progress_fn) {
+string bc0_compress(const string& data, ProgressCallback progress_fn) {
   return bc0_compress(data.data(), data.size(), progress_fn);
 }
 
-string bc0_compress(const void* in_data_v, size_t in_size, function<void(size_t, size_t)> progress_fn) {
+string bc0_compress(const void* in_data_v, size_t in_size, ProgressCallback progress_fn) {
   const uint8_t* in_data = reinterpret_cast<const uint8_t*>(in_data_v);
 
   LZSSInterleavedWriter w;
@@ -1106,7 +1104,7 @@ string bc0_compress(const void* in_data_v, size_t in_size, function<void(size_t,
   while (window.offset < in_size) {
     if (progress_fn && ((last_progress_fn_call_offset & ~0xFFF) != (window.offset & ~0xFFF))) {
       last_progress_fn_call_offset = window.offset;
-      progress_fn(window.offset, w.size());
+      progress_fn(CompressPhase::GENERATE_RESULT, window.offset, in_size, w.size());
     }
 
     auto match = window.get_best_match();

src/Compression.hh

@@ -10,19 +10,7 @@
 
 #include "Text.hh"
 
-enum class PRSCompressOptimalPhase {
+enum class CompressPhase {
-  INDEX_SHORT_COPIES = 0,
-  INDEX_LONG_COPIES,
-  INDEX_EXTENDED_COPIES,
-  CONSTRUCT_PATHS,
-  BACKTRACE_OPTIMAL_PATH,
-  GENERATE_RESULT,
-};
-
-template <>
-const char* name_for_enum<PRSCompressOptimalPhase>(PRSCompressOptimalPhase v);
-
-enum class BC0CompressOptimalPhase {
   INDEX = 0,
   CONSTRUCT_PATHS,
   BACKTRACE_OPTIMAL_PATH,
@@ -30,7 +18,9 @@ enum class BC0CompressOptimalPhase {
 };
 
 template <>
-const char* name_for_enum<BC0CompressOptimalPhase>(BC0CompressOptimalPhase v);
+const char* name_for_enum<CompressPhase>(CompressPhase v);
+
+typedef std::function<void(CompressPhase phase, size_t input_progress, size_t input_size, size_t output_size)> ProgressCallback;
 
 ////////////////////////////////////////////////////////////////////////////////
 // PRS compression
@@ -54,7 +44,7 @@ public:
   //       the backreference or ignoring it.
   //   2+: Consider further chains of paths at each point. Using values 2 or
   //       greater for compression_level generally yields diminishing returns.
-  explicit PRSCompressor(ssize_t compression_level = 0, std::function<void(size_t, size_t)> progress_fn = nullptr);
+  explicit PRSCompressor(ssize_t compression_level = 0, ProgressCallback progress_fn = nullptr);
   ~PRSCompressor() = default;
 
   // Adds more input data to be compressed, which logically comes after all
@@ -146,7 +136,7 @@ private:
   void flush_control();
 
   ssize_t compression_level;
-  std::function<void(size_t, size_t)> progress_fn;
+  ProgressCallback progress_fn;
   bool closed;
 
   size_t control_byte_offset;
@@ -166,20 +156,20 @@ std::string prs_compress(
     const void* vdata,
     size_t size,
     ssize_t compression_level = 0,
-    std::function<void(size_t, size_t)> progress_fn = nullptr);
+    ProgressCallback progress_fn = nullptr);
 std::string prs_compress(
     const std::string& data,
     ssize_t compression_level = 0,
-    std::function<void(size_t, size_t)> progress_fn = nullptr);
+    ProgressCallback progress_fn = nullptr);
 
 // A faster form of prs_compress that doesn't have a tunable compression level.
 std::string prs_compress_indexed(
     const void* vdata,
     size_t size,
-    std::function<void(size_t, size_t)> progress_fn = nullptr);
+    ProgressCallback progress_fn = nullptr);
 std::string prs_compress_indexed(
     const std::string& data,
-    std::function<void(size_t, size_t)> progress_fn = nullptr);
+    ProgressCallback progress_fn = nullptr);
 
 // Compresses data using PRS to the smallest possible output size. This function
 // is slow, but produces results significantly smaller than even Sega's original
@@ -187,7 +177,7 @@ std::string prs_compress_indexed(
 std::string prs_compress_optimal(
     const void* vdata,
     size_t size,
-    std::function<void(PRSCompressOptimalPhase, size_t, size_t)> progress_fn = nullptr);
+    ProgressCallback progress_fn = nullptr);
 
 // Decompresses PRS-compressed data.
 struct PRSDecompressResult {
@@ -217,9 +207,9 @@ void prs_disassemble(FILE* stream, const std::string& data);
 std::string bc0_compress_optimal(
     const void* in_data_v,
     size_t in_size,
-    std::function<void(BC0CompressOptimalPhase, size_t, size_t)> progress_fn = nullptr);
+    ProgressCallback progress_fn = nullptr);
-std::string bc0_compress(const std::string& data, std::function<void(size_t, size_t)> progress_fn = nullptr);
+std::string bc0_compress(const std::string& data, ProgressCallback progress_fn = nullptr);
-std::string bc0_compress(const void* in_data_v, size_t in_size, std::function<void(size_t, size_t)> progress_fn = nullptr);
+std::string bc0_compress(const void* in_data_v, size_t in_size, ProgressCallback progress_fn = nullptr);
 
 // Encodes data in a BC0-compatible format without compression (similar to using
 // compression_level=-1 with prs_compress).

src/Main.cc

@@ -608,13 +608,13 @@ int main(int argc, char** argv) {
       }
 
       size_t input_bytes = data.size();
-      auto progress_fn = [&](size_t input_progress, size_t output_progress) -> void {
+      auto progress_fn = [&](auto, size_t input_progress, size_t, size_t output_progress) -> void {
         float progress = static_cast<float>(input_progress * 100) / input_bytes;
         float size_ratio = static_cast<float>(output_progress * 100) / input_progress;
         fprintf(stderr, "... %zu/%zu (%g%%) => %zu (%g%%)    \r",
             input_progress, input_bytes, progress, output_progress, size_ratio);
       };
-      auto optimal_progress_fn = [&](auto phase, size_t input_progress, size_t output_progress) -> void {
+      auto optimal_progress_fn = [&](auto phase, size_t input_progress, size_t input_bytes, size_t output_progress) -> void {
         const char* phase_name = name_for_enum(phase);
         float progress = static_cast<float>(input_progress * 100) / input_bytes;
         float size_ratio = static_cast<float>(output_progress * 100) / input_progress;