diff --git a/.gitignore b/.gitignore index 1f6ea7c8..1d4fd0d8 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ release release.zip system/patch-bb/data system/patch-bb/psobb.pat -system/dol \ No newline at end of file +system/dol diff --git a/README.md b/README.md index e6cc5e62..2a75bad1 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ If you're not playing PSO Blue Burst on newserv, you can skip these steps. newserv implements a patch server for PSO PC and PSO BB game data. Any file or directory you put in the system/patch-bb or system/patch-pc directories will be synced to clients when they connect to the patch server. -To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums. +To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums. Also, in the case when checksums are cached, newserv may not actually load the data for a patch file until it's needed by a client. Therefore, modifying any part of the patch tree while newserv is running can cause clients to see an inconsistent view of it. For BB clients, newserv reads some files out of the patch data to implement game logic, so it's important that certain game files are synchronized between the server and the client. newserv contains defaults for these files in the system/blueburst/map directory, but if these don't match the client's copies of the files, odd behavior will occur in games. diff --git a/src/Main.cc b/src/Main.cc index 0cbcc174..5842c728 100644 --- a/src/Main.cc +++ b/src/Main.cc @@ -513,7 +513,7 @@ int main(int argc, char** argv) { state->bb_patch_file_index.reset(new PatchFileIndex("system/patch-bb")); try { auto gsl_file = state->bb_patch_file_index->get("./data/data.gsl"); - state->bb_data_gsl.reset(new GSLArchive(gsl_file->data)); + state->bb_data_gsl.reset(new GSLArchive(gsl_file->load_data())); config_log.info("data.gsl found in BB patch files"); } catch (const out_of_range&) { config_log.info("data.gsl is not present in BB patch files"); diff --git a/src/PatchFileIndex.cc b/src/PatchFileIndex.cc index d1c47b71..166a13ae 100644 --- a/src/PatchFileIndex.cc +++ b/src/PatchFileIndex.cc @@ -15,7 +15,21 @@ using namespace std; -PatchFileIndex::File::File() : crc32(0) { } +PatchFileIndex::File::File(PatchFileIndex* index) + : index(index), crc32(0), size(0) { } + +std::shared_ptr PatchFileIndex::File::load_data() { + if (!this->loaded_data) { + string relative_path = join(this->path_directories, "/") + "/" + this->name; + string full_path = this->index->root_dir + "/" + relative_path; + patch_index_log.info("Loading data for %s", relative_path.c_str()); + this->loaded_data.reset(new string(load_file(full_path))); + this->size = this->loaded_data->size(); + } + return this->loaded_data; +} + + PatchFileIndex::PatchFileIndex(const string& root_dir) : root_dir(root_dir) { @@ -58,12 +72,14 @@ PatchFileIndex::PatchFileIndex(const string& root_dir) auto st = stat(full_item_path); - shared_ptr f(new File()); + shared_ptr f(new File(this)); f->path_directories = path_directories; f->name = item; - int64_t file_crc32 = -1; + bool should_compute_crc32s = true; + shared_ptr cache_item_json; try { + cache_item_json = metadata_cache_json->at(relative_item_path); auto cache_item = metadata_cache_json->at(relative_item_path)->as_list(); uint64_t cached_size = cache_item.at(0)->as_int(); uint64_t cached_mtime = cache_item.at(1)->as_int(); @@ -73,34 +89,51 @@ PatchFileIndex::PatchFileIndex(const string& root_dir) if (static_cast(st.st_size) != cached_size) { throw runtime_error("file size has changed"); } - file_crc32 = cache_item.at(2)->as_int(); - patch_index_log.info("Using cached checksum for %s", relative_item_path.c_str()); + f->size = cached_size; + f->crc32 = cache_item.at(2)->as_int(); + for (const auto& chunk_crc32_json : cache_item.at(3)->as_list()) { + f->chunk_crcs.emplace_back(chunk_crc32_json->as_int()); + } + should_compute_crc32s = false; + patch_index_log.info("Using cached checksums for %s", relative_item_path.c_str()); } catch (const exception& e) { should_write_metadata_cache = true; - patch_index_log.info("Cannot use cached checksum for %s: %s", relative_item_path.c_str(), e.what()); + patch_index_log.info("Cannot use cached checksums for %s: %s", relative_item_path.c_str(), e.what()); } - string file_data = load_file(full_item_path); - f->data.reset(new string(move(file_data))); - f->crc32 = (file_crc32 < 0) - ? ::crc32(f->data->data(), f->data->size()) : file_crc32; - for (size_t x = 0; x < f->data->size(); x += 0x4000) { - size_t chunk_bytes = min(f->data->size() - x, 0x4000); - f->chunk_crcs.emplace_back(::crc32(f->data->data() + x, chunk_bytes)); - } + if (should_compute_crc32s) { + auto data = f->load_data(); // Sets f->size + f->crc32 = crc32(data->data(), f->size); + for (size_t x = 0; x < data->size(); x += 0x4000) { + size_t chunk_bytes = min(f->size - x, 0x4000); + f->chunk_crcs.emplace_back(::crc32(data->data() + x, chunk_bytes)); + } - vector> new_cache_item({ - make_json_int(f->data->size()), - make_json_int(st.st_mtime), - make_json_int(f->crc32), - }); - new_metadata_cache_json->as_dict().emplace( - relative_item_path, make_json_list(move(new_cache_item))); + // File was modified or cache item was missing; make a new cache item + vector> chunk_crcs_item; + for (uint32_t chunk_crc : f->chunk_crcs) { + chunk_crcs_item.emplace_back(make_json_int(chunk_crc)); + } + vector> new_cache_item({ + make_json_int(f->size), + make_json_int(st.st_mtime), + make_json_int(f->crc32), + make_json_list(move(chunk_crcs_item)), + }); + new_metadata_cache_json->as_dict().emplace( + relative_item_path, make_json_list(move(new_cache_item))); + + } else { + // File was not modified and cache item was valid; just use the + // existing cache item + new_metadata_cache_json->as_dict().emplace( + relative_item_path, cache_item_json); + } this->files_by_patch_order.emplace_back(f); this->files_by_name.emplace(relative_item_path, f); - patch_index_log.info("Added file %s (%zu bytes; %zu chunks; %08" PRIX32 ")", - full_item_path.c_str(), f->data->size(), f->chunk_crcs.size(), f->crc32); + patch_index_log.info("Added file %s (%" PRIu32 " bytes; %zu chunks; %08" PRIX32 ")", + full_item_path.c_str(), f->size, f->chunk_crcs.size(), f->crc32); } } @@ -121,12 +154,12 @@ PatchFileIndex::PatchFileIndex(const string& root_dir) } } -const vector>& +const vector>& PatchFileIndex::all_files() const { return this->files_by_patch_order; } -shared_ptr PatchFileIndex::get( +shared_ptr PatchFileIndex::get( const string& filename) const { return this->files_by_name.at(filename); } diff --git a/src/PatchFileIndex.hh b/src/PatchFileIndex.hh index 53535599..0f6ac7e7 100644 --- a/src/PatchFileIndex.hh +++ b/src/PatchFileIndex.hh @@ -14,35 +14,38 @@ struct PatchFileIndex { explicit PatchFileIndex(const std::string& root_dir); struct File { + PatchFileIndex* index; std::vector path_directories; std::string name; - std::shared_ptr data; + std::shared_ptr loaded_data; std::vector chunk_crcs; uint32_t crc32; + uint32_t size; - File(); + explicit File(PatchFileIndex* index); + std::shared_ptr load_data(); }; - const std::vector>& all_files() const; - std::shared_ptr get(const std::string& filename) const; + const std::vector>& all_files() const; + std::shared_ptr get(const std::string& filename) const; private: - std::vector> files_by_patch_order; - std::unordered_map> files_by_name; + std::vector> files_by_patch_order; + std::unordered_map> files_by_name; std::string root_dir; }; struct PatchFileChecksumRequest { - std::shared_ptr file; + std::shared_ptr file; uint32_t crc32; uint32_t size; bool response_received; - explicit PatchFileChecksumRequest(std::shared_ptr file) + explicit PatchFileChecksumRequest(std::shared_ptr file) : file(file), crc32(0), size(0), response_received(false) { } inline bool needs_update() const { return !this->response_received || (this->crc32 != this->file->crc32) || - (this->size != this->file->data->size()); + (this->size != this->file->size); } }; diff --git a/src/ReceiveCommands.cc b/src/ReceiveCommands.cc index 7ba9fd45..cff43a3c 100644 --- a/src/ReceiveCommands.cc +++ b/src/ReceiveCommands.cc @@ -2621,9 +2621,9 @@ static void on_checksums_done_patch(shared_ptr, throw runtime_error("client did not respond to checksum request"); } if (req.needs_update()) { - c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %zu/%" PRIu32 ")", - req.file->name.c_str(), req.file->crc32, req.crc32, req.file->data->size(), req.size); - start_cmd.total_bytes += req.file->data->size(); + c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %" PRIu32 "/%" PRIu32 ")", + req.file->name.c_str(), req.file->crc32, req.crc32, req.file->size, req.size); + start_cmd.total_bytes += req.file->size; start_cmd.num_files++; } else { c->log.info("File %s is up to date", req.file->name.c_str()); diff --git a/src/SendCommands.cc b/src/SendCommands.cc index 60502e69..e6a9e472 100644 --- a/src/SendCommands.cc +++ b/src/SendCommands.cc @@ -471,8 +471,8 @@ void send_enter_directory_patch(shared_ptr c, const string& dir) { send_command_t(c, 0x09, 0x00, cmd); } -void send_patch_file(shared_ptr c, shared_ptr f) { - S_OpenFile_Patch_06 open_cmd = {0, f->data->size(), f->name}; +void send_patch_file(shared_ptr c, shared_ptr f) { + S_OpenFile_Patch_06 open_cmd = {0, f->size, f->name}; send_command_t(c, 0x06, 0x00, open_cmd); for (size_t x = 0; x < f->chunk_crcs.size(); x++) { @@ -480,11 +480,12 @@ void send_patch_file(shared_ptr c, shared_ptr(f->data->size() - x * 0x4000, 0x4000); + auto data = f->load_data(); + size_t chunk_size = min(f->size - (x * 0x4000), 0x4000); S_WriteFileHeader_Patch_07 write_cmd_header = { x, f->chunk_crcs[x], chunk_size}; w.put(write_cmd_header); - w.write(f->data->data() + x * 0x4000, chunk_size); + w.write(data->data() + (x * 0x4000), chunk_size); while (w.size() & 7) { w.put_u8(0); } diff --git a/src/SendCommands.hh b/src/SendCommands.hh index 5e1d0ac1..63d6fe62 100644 --- a/src/SendCommands.hh +++ b/src/SendCommands.hh @@ -142,7 +142,7 @@ void send_approve_player_choice_bb(std::shared_ptr c); void send_complete_player_bb(std::shared_ptr c); void send_enter_directory_patch(std::shared_ptr c, const std::string& dir); -void send_patch_file(std::shared_ptr c, std::shared_ptr f); +void send_patch_file(std::shared_ptr c, std::shared_ptr f); void send_message_box(std::shared_ptr c, const std::u16string& text); void send_lobby_name(std::shared_ptr c, const std::u16string& text); diff --git a/src/ServerState.cc b/src/ServerState.cc index 50e3b2b3..60b1dd19 100644 --- a/src/ServerState.cc +++ b/src/ServerState.cc @@ -436,7 +436,7 @@ shared_ptr ServerState::load_bb_file( // First, look in the patch tree's data directory string patch_index_path = "./data/" + patch_index_filename; try { - auto ret = this->bb_patch_file_index->get(patch_index_path)->data; + auto ret = this->bb_patch_file_index->get(patch_index_path)->load_data(); static_game_data_log.info("Loaded %s from file in BB patch tree", patch_index_path.c_str()); return ret; } catch (const out_of_range&) {