From 176e0fb6d6d622205412123597b0d5e5a7e70624 Mon Sep 17 00:00:00 2001 From: Martin Michelsen Date: Sat, 3 Sep 2022 22:53:33 -0700 Subject: [PATCH] cache file checksums in patch trees --- .gitignore | 2 ++ README.md | 2 ++ src/PatchFileIndex.cc | 80 +++++++++++++++++++++++++++++++++++-------- src/PatchFileIndex.hh | 1 - 4 files changed, 70 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 8cc981cb..1f6ea7c8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ system/licenses.nsi system/players/player_* system/players/account_* system/players/bank_* +system/patch-pc/.metadata-cache.json +system/patch-bb/.metadata-cache.json # Files fuzziqersoftware uses that don't make sense to be committed to the main # repository diff --git a/README.md b/README.md index 4664bac4..e6cc5e62 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,8 @@ If you're not playing PSO Blue Burst on newserv, you can skip these steps. newserv implements a patch server for PSO PC and PSO BB game data. Any file or directory you put in the system/patch-bb or system/patch-pc directories will be synced to clients when they connect to the patch server. +To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums. + For BB clients, newserv reads some files out of the patch data to implement game logic, so it's important that certain game files are synchronized between the server and the client. newserv contains defaults for these files in the system/blueburst/map directory, but if these don't match the client's copies of the files, odd behavior will occur in games. Specifically, the patch-bb directory should contain at least the data.gsl file and all map_*.dat files from the version of PSOBB that you want to play on newserv. You can copy these files out of the client's data directory from a clean installation, and put them in system/patch-bb/data. diff --git a/src/PatchFileIndex.cc b/src/PatchFileIndex.cc index f6d4f5fe..d1c47b71 100644 --- a/src/PatchFileIndex.cc +++ b/src/PatchFileIndex.cc @@ -17,22 +17,25 @@ using namespace std; PatchFileIndex::File::File() : crc32(0) { } -void PatchFileIndex::File::load_data(const string& root_dir) { - string full_path = root_dir + '/' + join(this->path_directories, "/") + '/' + this->name; - auto f = fopen_unique(full_path, "rb"); - - string file_data = load_file(full_path); - this->data.reset(new string(move(file_data))); - this->crc32 = ::crc32(this->data->data(), this->data->size()); - for (size_t x = 0; x < this->data->size(); x += 0x4000) { - size_t chunk_bytes = min(this->data->size() - x, 0x4000); - this->chunk_crcs.emplace_back(::crc32(this->data->data() + x, chunk_bytes)); - } -} - PatchFileIndex::PatchFileIndex(const string& root_dir) : root_dir(root_dir) { + string metadata_cache_filename = root_dir + "/.metadata-cache.json"; + shared_ptr metadata_cache_json; + try { + string metadata_text = load_file(metadata_cache_filename); + metadata_cache_json = JSONObject::parse(metadata_text); + patch_index_log.info("Loaded patch metadata cache from %s", metadata_cache_filename.c_str()); + } catch (const exception& e) { + metadata_cache_json = make_json_dict({}); + patch_index_log.warning("Cannot load patch metadata cache from %s: %s", metadata_cache_filename.c_str(), e.what()); + } + + // Assuming it's rare for patch files to change, we skip writing the metadata + // cache if no files were changed at all (which should usually be the case) + bool should_write_metadata_cache = false; + shared_ptr new_metadata_cache_json = make_json_dict({}); + vector path_directories; function collect_dir = [&](const string& dir) -> void { path_directories.emplace_back(dir); @@ -52,10 +55,48 @@ PatchFileIndex::PatchFileIndex(const string& root_dir) if (isdir(full_item_path)) { collect_dir(item); } else if (isfile(full_item_path)) { + + auto st = stat(full_item_path); + shared_ptr f(new File()); f->path_directories = path_directories; f->name = item; - f->load_data(root_dir); + + int64_t file_crc32 = -1; + try { + auto cache_item = metadata_cache_json->at(relative_item_path)->as_list(); + uint64_t cached_size = cache_item.at(0)->as_int(); + uint64_t cached_mtime = cache_item.at(1)->as_int(); + if (static_cast(st.st_mtime) != cached_mtime) { + throw runtime_error("file has been modified"); + } + if (static_cast(st.st_size) != cached_size) { + throw runtime_error("file size has changed"); + } + file_crc32 = cache_item.at(2)->as_int(); + patch_index_log.info("Using cached checksum for %s", relative_item_path.c_str()); + } catch (const exception& e) { + should_write_metadata_cache = true; + patch_index_log.info("Cannot use cached checksum for %s: %s", relative_item_path.c_str(), e.what()); + } + + string file_data = load_file(full_item_path); + f->data.reset(new string(move(file_data))); + f->crc32 = (file_crc32 < 0) + ? ::crc32(f->data->data(), f->data->size()) : file_crc32; + for (size_t x = 0; x < f->data->size(); x += 0x4000) { + size_t chunk_bytes = min(f->data->size() - x, 0x4000); + f->chunk_crcs.emplace_back(::crc32(f->data->data() + x, chunk_bytes)); + } + + vector> new_cache_item({ + make_json_int(f->data->size()), + make_json_int(st.st_mtime), + make_json_int(f->crc32), + }); + new_metadata_cache_json->as_dict().emplace( + relative_item_path, make_json_list(move(new_cache_item))); + this->files_by_patch_order.emplace_back(f); this->files_by_name.emplace(relative_item_path, f); patch_index_log.info("Added file %s (%zu bytes; %zu chunks; %08" PRIX32 ")", @@ -67,6 +108,17 @@ PatchFileIndex::PatchFileIndex(const string& root_dir) }; collect_dir("."); + + if (should_write_metadata_cache) { + try { + save_file(metadata_cache_filename, new_metadata_cache_json->serialize()); + patch_index_log.info("Saved patch metadata cache to %s", metadata_cache_filename.c_str()); + } catch (const exception& e) { + patch_index_log.warning("Cannot save patch metadata cache to %s: %s", metadata_cache_filename.c_str(), e.what()); + } + } else { + patch_index_log.info("No files were modified; skipping metadata cache update"); + } } const vector>& diff --git a/src/PatchFileIndex.hh b/src/PatchFileIndex.hh index 8526d3db..53535599 100644 --- a/src/PatchFileIndex.hh +++ b/src/PatchFileIndex.hh @@ -21,7 +21,6 @@ struct PatchFileIndex { uint32_t crc32; File(); - void load_data(const std::string& root_dir); }; const std::vector>& all_files() const;