also store chunk crcs in patch metadata cache

.gitignore

@@ -33,4 +33,4 @@ release
 release.zip
 system/patch-bb/data
 system/patch-bb/psobb.pat
-system/dol
+system/dol

README.md

@@ -94,7 +94,7 @@ If you're not playing PSO Blue Burst on newserv, you can skip these steps.
 
 newserv implements a patch server for PSO PC and PSO BB game data. Any file or directory you put in the system/patch-bb or system/patch-pc directories will be synced to clients when they connect to the patch server.
 
-To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums.
+To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums. Also, in the case when checksums are cached, newserv may not actually load the data for a patch file until it's needed by a client. Therefore, modifying any part of the patch tree while newserv is running can cause clients to see an inconsistent view of it.
 
 For BB clients, newserv reads some files out of the patch data to implement game logic, so it's important that certain game files are synchronized between the server and the client. newserv contains defaults for these files in the system/blueburst/map directory, but if these don't match the client's copies of the files, odd behavior will occur in games.
 

src/Main.cc

@@ -513,7 +513,7 @@ int main(int argc, char** argv) {
         state->bb_patch_file_index.reset(new PatchFileIndex("system/patch-bb"));
         try {
           auto gsl_file = state->bb_patch_file_index->get("./data/data.gsl");
-          state->bb_data_gsl.reset(new GSLArchive(gsl_file->data));
+          state->bb_data_gsl.reset(new GSLArchive(gsl_file->load_data()));
           config_log.info("data.gsl found in BB patch files");
         } catch (const out_of_range&) {
           config_log.info("data.gsl is not present in BB patch files");

src/PatchFileIndex.cc

@@ -15,7 +15,21 @@ using namespace std;
 
 
 
-PatchFileIndex::File::File() : crc32(0) { }
+PatchFileIndex::File::File(PatchFileIndex* index)
+  : index(index), crc32(0), size(0) { }
+
+std::shared_ptr<const std::string> PatchFileIndex::File::load_data() {
+  if (!this->loaded_data) {
+    string relative_path = join(this->path_directories, "/") + "/" + this->name;
+    string full_path = this->index->root_dir + "/" + relative_path;
+    patch_index_log.info("Loading data for %s", relative_path.c_str());
+    this->loaded_data.reset(new string(load_file(full_path)));
+    this->size = this->loaded_data->size();
+  }
+  return this->loaded_data;
+}
+
+
 
 PatchFileIndex::PatchFileIndex(const string& root_dir)
   : root_dir(root_dir) {
@@ -58,12 +72,14 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
 
         auto st = stat(full_item_path);
 
-        shared_ptr<File> f(new File());
+        shared_ptr<File> f(new File(this));
         f->path_directories = path_directories;
         f->name = item;
 
-        int64_t file_crc32 = -1;
+        bool should_compute_crc32s = true;
+        shared_ptr<JSONObject> cache_item_json;
         try {
+          cache_item_json = metadata_cache_json->at(relative_item_path);
           auto cache_item = metadata_cache_json->at(relative_item_path)->as_list();
           uint64_t cached_size = cache_item.at(0)->as_int();
           uint64_t cached_mtime = cache_item.at(1)->as_int();
@@ -73,34 +89,51 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
           if (static_cast<uint64_t>(st.st_size) != cached_size) {
             throw runtime_error("file size has changed");
           }
-          file_crc32 = cache_item.at(2)->as_int();
+          f->size = cached_size;
-          patch_index_log.info("Using cached checksum for %s", relative_item_path.c_str());
+          f->crc32 = cache_item.at(2)->as_int();
+          for (const auto& chunk_crc32_json : cache_item.at(3)->as_list()) {
+            f->chunk_crcs.emplace_back(chunk_crc32_json->as_int());
+          }
+          should_compute_crc32s = false;
+          patch_index_log.info("Using cached checksums for %s", relative_item_path.c_str());
         } catch (const exception& e) {
           should_write_metadata_cache = true;
-          patch_index_log.info("Cannot use cached checksum for %s: %s", relative_item_path.c_str(), e.what());
+          patch_index_log.info("Cannot use cached checksums for %s: %s", relative_item_path.c_str(), e.what());
         }
 
-        string file_data = load_file(full_item_path);
+        if (should_compute_crc32s) {
-        f->data.reset(new string(move(file_data)));
+          auto data = f->load_data(); // Sets f->size
-        f->crc32 = (file_crc32 < 0)
+          f->crc32 = crc32(data->data(), f->size);
-            ? ::crc32(f->data->data(), f->data->size()) : file_crc32;
+          for (size_t x = 0; x < data->size(); x += 0x4000) {
-        for (size_t x = 0; x < f->data->size(); x += 0x4000) {
+            size_t chunk_bytes = min<size_t>(f->size - x, 0x4000);
-          size_t chunk_bytes = min<size_t>(f->data->size() - x, 0x4000);
+            f->chunk_crcs.emplace_back(::crc32(data->data() + x, chunk_bytes));
-          f->chunk_crcs.emplace_back(::crc32(f->data->data() + x, chunk_bytes));
+          }
-        }
 
-        vector<shared_ptr<JSONObject>> new_cache_item({
+          // File was modified or cache item was missing; make a new cache item
-          make_json_int(f->data->size()),
+          vector<shared_ptr<JSONObject>> chunk_crcs_item;
-          make_json_int(st.st_mtime),
+          for (uint32_t chunk_crc : f->chunk_crcs) {
-          make_json_int(f->crc32),
+            chunk_crcs_item.emplace_back(make_json_int(chunk_crc));
-        });
+          }
-        new_metadata_cache_json->as_dict().emplace(
+          vector<shared_ptr<JSONObject>> new_cache_item({
-            relative_item_path, make_json_list(move(new_cache_item)));
+            make_json_int(f->size),
+            make_json_int(st.st_mtime),
+            make_json_int(f->crc32),
+            make_json_list(move(chunk_crcs_item)),
+          });
+          new_metadata_cache_json->as_dict().emplace(
+              relative_item_path, make_json_list(move(new_cache_item)));
+
+        } else {
+          // File was not modified and cache item was valid; just use the
+          // existing cache item
+          new_metadata_cache_json->as_dict().emplace(
+              relative_item_path, cache_item_json);
+        }
 
         this->files_by_patch_order.emplace_back(f);
         this->files_by_name.emplace(relative_item_path, f);
-        patch_index_log.info("Added file %s (%zu bytes; %zu chunks; %08" PRIX32 ")",
+        patch_index_log.info("Added file %s (%" PRIu32 " bytes; %zu chunks; %08" PRIX32 ")",
-            full_item_path.c_str(), f->data->size(), f->chunk_crcs.size(), f->crc32);
+            full_item_path.c_str(), f->size, f->chunk_crcs.size(), f->crc32);
       }
     }
 
@@ -121,12 +154,12 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
   }
 }
 
-const vector<shared_ptr<const PatchFileIndex::File>>&
+const vector<shared_ptr<PatchFileIndex::File>>&
 PatchFileIndex::all_files() const {
   return this->files_by_patch_order;
 }
 
-shared_ptr<const PatchFileIndex::File> PatchFileIndex::get(
+shared_ptr<PatchFileIndex::File> PatchFileIndex::get(
     const string& filename) const {
   return this->files_by_name.at(filename);
 }

src/PatchFileIndex.hh

@@ -14,35 +14,38 @@ struct PatchFileIndex {
   explicit PatchFileIndex(const std::string& root_dir);
 
   struct File {
+    PatchFileIndex* index;
     std::vector<std::string> path_directories;
     std::string name;
-    std::shared_ptr<const std::string> data;
+    std::shared_ptr<const std::string> loaded_data;
     std::vector<uint32_t> chunk_crcs;
     uint32_t crc32;
+    uint32_t size;
 
-    File();
+    explicit File(PatchFileIndex* index);
+    std::shared_ptr<const std::string> load_data();
   };
 
-  const std::vector<std::shared_ptr<const File>>& all_files() const;
+  const std::vector<std::shared_ptr<File>>& all_files() const;
-  std::shared_ptr<const File> get(const std::string& filename) const;
+  std::shared_ptr<File> get(const std::string& filename) const;
 
 private:
-  std::vector<std::shared_ptr<const File>> files_by_patch_order;
+  std::vector<std::shared_ptr<File>> files_by_patch_order;
-  std::unordered_map<std::string, std::shared_ptr<const File>> files_by_name;
+  std::unordered_map<std::string, std::shared_ptr<File>> files_by_name;
   std::string root_dir;
 };
 
 struct PatchFileChecksumRequest {
-  std::shared_ptr<const PatchFileIndex::File> file;
+  std::shared_ptr<PatchFileIndex::File> file;
   uint32_t crc32;
   uint32_t size;
   bool response_received;
 
-  explicit PatchFileChecksumRequest(std::shared_ptr<const PatchFileIndex::File> file)
+  explicit PatchFileChecksumRequest(std::shared_ptr<PatchFileIndex::File> file)
     : file(file), crc32(0), size(0), response_received(false) { }
   inline bool needs_update() const {
     return !this->response_received ||
            (this->crc32 != this->file->crc32) ||
-           (this->size != this->file->data->size());
+           (this->size != this->file->size);
   }
 };

src/ReceiveCommands.cc

@@ -2621,9 +2621,9 @@ static void on_checksums_done_patch(shared_ptr<ServerState>,
       throw runtime_error("client did not respond to checksum request");
     }
     if (req.needs_update()) {
-      c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %zu/%" PRIu32 ")",
+      c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %" PRIu32 "/%" PRIu32 ")",
-          req.file->name.c_str(), req.file->crc32, req.crc32, req.file->data->size(), req.size);
+          req.file->name.c_str(), req.file->crc32, req.crc32, req.file->size, req.size);
-      start_cmd.total_bytes += req.file->data->size();
+      start_cmd.total_bytes += req.file->size;
       start_cmd.num_files++;
     } else {
       c->log.info("File %s is up to date", req.file->name.c_str());

src/SendCommands.cc

@@ -471,8 +471,8 @@ void send_enter_directory_patch(shared_ptr<Client> c, const string& dir) {
   send_command_t(c, 0x09, 0x00, cmd);
 }
 
-void send_patch_file(shared_ptr<Client> c, shared_ptr<const PatchFileIndex::File> f) {
+void send_patch_file(shared_ptr<Client> c, shared_ptr<PatchFileIndex::File> f) {
-  S_OpenFile_Patch_06 open_cmd = {0, f->data->size(), f->name};
+  S_OpenFile_Patch_06 open_cmd = {0, f->size, f->name};
   send_command_t(c, 0x06, 0x00, open_cmd);
 
   for (size_t x = 0; x < f->chunk_crcs.size(); x++) {
@@ -480,11 +480,12 @@ void send_patch_file(shared_ptr<Client> c, shared_ptr<const PatchFileIndex::File
     // Channel::send that takes iovecs or something to avoid these dumb massive
     // string copies.
     StringWriter w;
-    size_t chunk_size = min<uint32_t>(f->data->size() - x * 0x4000, 0x4000);
+    auto data = f->load_data();
+    size_t chunk_size = min<uint32_t>(f->size - (x * 0x4000), 0x4000);
     S_WriteFileHeader_Patch_07 write_cmd_header = {
         x, f->chunk_crcs[x], chunk_size};
     w.put(write_cmd_header);
-    w.write(f->data->data() + x * 0x4000, chunk_size);
+    w.write(data->data() + (x * 0x4000), chunk_size);
     while (w.size() & 7) {
       w.put_u8(0);
     }

src/SendCommands.hh

@@ -142,7 +142,7 @@ void send_approve_player_choice_bb(std::shared_ptr<Client> c);
 void send_complete_player_bb(std::shared_ptr<Client> c);
 
 void send_enter_directory_patch(std::shared_ptr<Client> c, const std::string& dir);
-void send_patch_file(std::shared_ptr<Client> c, std::shared_ptr<const PatchFileIndex::File> f);
+void send_patch_file(std::shared_ptr<Client> c, std::shared_ptr<PatchFileIndex::File> f);
 
 void send_message_box(std::shared_ptr<Client> c, const std::u16string& text);
 void send_lobby_name(std::shared_ptr<Client> c, const std::u16string& text);

src/ServerState.cc

@@ -436,7 +436,7 @@ shared_ptr<const string> ServerState::load_bb_file(
     // First, look in the patch tree's data directory
     string patch_index_path = "./data/" + patch_index_filename;
     try {
-      auto ret = this->bb_patch_file_index->get(patch_index_path)->data;
+      auto ret = this->bb_patch_file_index->get(patch_index_path)->load_data();
       static_game_data_log.info("Loaded %s from file in BB patch tree", patch_index_path.c_str());
       return ret;
     } catch (const out_of_range&) {