cache file checksums in patch trees

.gitignore

@@ -19,6 +19,8 @@ system/licenses.nsi
 system/players/player_*
 system/players/account_*
 system/players/bank_*
+system/patch-pc/.metadata-cache.json
+system/patch-bb/.metadata-cache.json
 
 # Files fuzziqersoftware uses that don't make sense to be committed to the main
 # repository

README.md

@@ -94,6 +94,8 @@ If you're not playing PSO Blue Burst on newserv, you can skip these steps.
 
 newserv implements a patch server for PSO PC and PSO BB game data. Any file or directory you put in the system/patch-bb or system/patch-pc directories will be synced to clients when they connect to the patch server.
 
+To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums.
+
 For BB clients, newserv reads some files out of the patch data to implement game logic, so it's important that certain game files are synchronized between the server and the client. newserv contains defaults for these files in the system/blueburst/map directory, but if these don't match the client's copies of the files, odd behavior will occur in games.
 
 Specifically, the patch-bb directory should contain at least the data.gsl file and all map_*.dat files from the version of PSOBB that you want to play on newserv. You can copy these files out of the client's data directory from a clean installation, and put them in system/patch-bb/data.

src/PatchFileIndex.cc

@@ -17,22 +17,25 @@ using namespace std;
 
 PatchFileIndex::File::File() : crc32(0) { }
 
-void PatchFileIndex::File::load_data(const string& root_dir) {
-  string full_path = root_dir + '/' + join(this->path_directories, "/") + '/' + this->name;
-  auto f = fopen_unique(full_path, "rb");
-
-  string file_data = load_file(full_path);
-  this->data.reset(new string(move(file_data)));
-  this->crc32 = ::crc32(this->data->data(), this->data->size());
-  for (size_t x = 0; x < this->data->size(); x += 0x4000) {
-    size_t chunk_bytes = min<size_t>(this->data->size() - x, 0x4000);
-    this->chunk_crcs.emplace_back(::crc32(this->data->data() + x, chunk_bytes));
-  }
-}
-
 PatchFileIndex::PatchFileIndex(const string& root_dir)
   : root_dir(root_dir) {
 
+  string metadata_cache_filename = root_dir + "/.metadata-cache.json";
+  shared_ptr<JSONObject> metadata_cache_json;
+  try {
+    string metadata_text = load_file(metadata_cache_filename);
+    metadata_cache_json = JSONObject::parse(metadata_text);
+    patch_index_log.info("Loaded patch metadata cache from %s", metadata_cache_filename.c_str());
+  } catch (const exception& e) {
+    metadata_cache_json = make_json_dict({});
+    patch_index_log.warning("Cannot load patch metadata cache from %s: %s", metadata_cache_filename.c_str(), e.what());
+  }
+
+  // Assuming it's rare for patch files to change, we skip writing the metadata
+  // cache if no files were changed at all (which should usually be the case)
+  bool should_write_metadata_cache = false;
+  shared_ptr<JSONObject> new_metadata_cache_json = make_json_dict({});
+
   vector<string> path_directories;
   function<void(const string&)> collect_dir = [&](const string& dir) -> void {
     path_directories.emplace_back(dir);
@@ -52,10 +55,48 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
       if (isdir(full_item_path)) {
         collect_dir(item);
       } else if (isfile(full_item_path)) {
+
+        auto st = stat(full_item_path);
+
         shared_ptr<File> f(new File());
         f->path_directories = path_directories;
         f->name = item;
-        f->load_data(root_dir);
+
+        int64_t file_crc32 = -1;
+        try {
+          auto cache_item = metadata_cache_json->at(relative_item_path)->as_list();
+          uint64_t cached_size = cache_item.at(0)->as_int();
+          uint64_t cached_mtime = cache_item.at(1)->as_int();
+          if (static_cast<uint64_t>(st.st_mtime) != cached_mtime) {
+            throw runtime_error("file has been modified");
+          }
+          if (static_cast<uint64_t>(st.st_size) != cached_size) {
+            throw runtime_error("file size has changed");
+          }
+          file_crc32 = cache_item.at(2)->as_int();
+          patch_index_log.info("Using cached checksum for %s", relative_item_path.c_str());
+        } catch (const exception& e) {
+          should_write_metadata_cache = true;
+          patch_index_log.info("Cannot use cached checksum for %s: %s", relative_item_path.c_str(), e.what());
+        }
+
+        string file_data = load_file(full_item_path);
+        f->data.reset(new string(move(file_data)));
+        f->crc32 = (file_crc32 < 0)
+            ? ::crc32(f->data->data(), f->data->size()) : file_crc32;
+        for (size_t x = 0; x < f->data->size(); x += 0x4000) {
+          size_t chunk_bytes = min<size_t>(f->data->size() - x, 0x4000);
+          f->chunk_crcs.emplace_back(::crc32(f->data->data() + x, chunk_bytes));
+        }
+
+        vector<shared_ptr<JSONObject>> new_cache_item({
+          make_json_int(f->data->size()),
+          make_json_int(st.st_mtime),
+          make_json_int(f->crc32),
+        });
+        new_metadata_cache_json->as_dict().emplace(
+            relative_item_path, make_json_list(move(new_cache_item)));
+
         this->files_by_patch_order.emplace_back(f);
         this->files_by_name.emplace(relative_item_path, f);
         patch_index_log.info("Added file %s (%zu bytes; %zu chunks; %08" PRIX32 ")",
@@ -67,6 +108,17 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
   };
 
   collect_dir(".");
+
+  if (should_write_metadata_cache) {
+    try {
+      save_file(metadata_cache_filename, new_metadata_cache_json->serialize());
+      patch_index_log.info("Saved patch metadata cache to %s", metadata_cache_filename.c_str());
+    } catch (const exception& e) {
+      patch_index_log.warning("Cannot save patch metadata cache to %s: %s", metadata_cache_filename.c_str(), e.what());
+    }
+  } else {
+    patch_index_log.info("No files were modified; skipping metadata cache update");
+  }
 }
 
 const vector<shared_ptr<const PatchFileIndex::File>>&

src/PatchFileIndex.hh

@@ -21,7 +21,6 @@ struct PatchFileIndex {
     uint32_t crc32;
 
     File();
-    void load_data(const std::string& root_dir);
   };
 
   const std::vector<std::shared_ptr<const File>>& all_files() const;