incentive/psopeeps-newserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

also store chunk crcs in patch metadata cache

Browse Source
This commit is contained in:
Martin Michelsen
2022-09-03 23:24:35 -07:00
parent f166dae1c6
commit 764a930213
9 changed files with 83 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -1
View File
@@ -33,4 +33,4 @@ release
release.zip
system/patch-bb/data
system/patch-bb/psobb.pat
system/dol
system/dol
README.md
+1 -1
View File
@@ -94,7 +94,7 @@ If you're not playing PSO Blue Burst on newserv, you can skip these steps.
newserv implements a patch server for PSO PC and PSO BB game data. Any file or directory you put in the system/patch-bb or system/patch-pc directories will be synced to clients when they connect to the patch server.
To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums.
To make server startup faster, newserv caches the modification times, sizes, and checksums of the files in the patch directories. If the patch server appears to be misbehaving, try deleting the .metadata-cache.json file in the relevant patch directory to force newserv to recompute all the checksums. Also, in the case when checksums are cached, newserv may not actually load the data for a patch file until it's needed by a client. Therefore, modifying any part of the patch tree while newserv is running can cause clients to see an inconsistent view of it.
For BB clients, newserv reads some files out of the patch data to implement game logic, so it's important that certain game files are synchronized between the server and the client. newserv contains defaults for these files in the system/blueburst/map directory, but if these don't match the client's copies of the files, odd behavior will occur in games.
src/Main.cc
+1 -1
View File
@@ -513,7 +513,7 @@ int main(int argc, char** argv) {
state->bb_patch_file_index.reset(new PatchFileIndex("system/patch-bb"));
try {
auto gsl_file = state->bb_patch_file_index->get("./data/data.gsl");
state->bb_data_gsl.reset(new GSLArchive(gsl_file->data));
state->bb_data_gsl.reset(new GSLArchive(gsl_file->load_data()));
config_log.info("data.gsl found in BB patch files");
} catch (const out_of_range&) {
config_log.info("data.gsl is not present in BB patch files");
src/PatchFileIndex.cc
+58 -25
View File
@@ -15,7 +15,21 @@ using namespace std;
PatchFileIndex::File::File() : crc32(0) { }
PatchFileIndex::File::File(PatchFileIndex* index)
: index(index), crc32(0), size(0) { }
std::shared_ptr<const std::string> PatchFileIndex::File::load_data() {
if (!this->loaded_data) {
string relative_path = join(this->path_directories, "/") + "/" + this->name;
string full_path = this->index->root_dir + "/" + relative_path;
patch_index_log.info("Loading data for %s", relative_path.c_str());
this->loaded_data.reset(new string(load_file(full_path)));
this->size = this->loaded_data->size();
}
return this->loaded_data;
}
PatchFileIndex::PatchFileIndex(const string& root_dir)
: root_dir(root_dir) {
@@ -58,12 +72,14 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
auto st = stat(full_item_path);
shared_ptr<File> f(new File());
shared_ptr<File> f(new File(this));
f->path_directories = path_directories;
f->name = item;
int64_t file_crc32 = -1;
bool should_compute_crc32s = true;
shared_ptr<JSONObject> cache_item_json;
try {
cache_item_json = metadata_cache_json->at(relative_item_path);
auto cache_item = metadata_cache_json->at(relative_item_path)->as_list();
uint64_t cached_size = cache_item.at(0)->as_int();
uint64_t cached_mtime = cache_item.at(1)->as_int();
@@ -73,34 +89,51 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
if (static_cast<uint64_t>(st.st_size) != cached_size) {
throw runtime_error("file size has changed");
}
file_crc32 = cache_item.at(2)->as_int();
patch_index_log.info("Using cached checksum for %s", relative_item_path.c_str());
f->size = cached_size;
f->crc32 = cache_item.at(2)->as_int();
for (const auto& chunk_crc32_json : cache_item.at(3)->as_list()) {
f->chunk_crcs.emplace_back(chunk_crc32_json->as_int());
}
should_compute_crc32s = false;
patch_index_log.info("Using cached checksums for %s", relative_item_path.c_str());
} catch (const exception& e) {
should_write_metadata_cache = true;
patch_index_log.info("Cannot use cached checksum for %s: %s", relative_item_path.c_str(), e.what());
patch_index_log.info("Cannot use cached checksums for %s: %s", relative_item_path.c_str(), e.what());
}
string file_data = load_file(full_item_path);
f->data.reset(new string(move(file_data)));
f->crc32 = (file_crc32 < 0)
? ::crc32(f->data->data(), f->data->size()) : file_crc32;
for (size_t x = 0; x < f->data->size(); x += 0x4000) {
size_t chunk_bytes = min<size_t>(f->data->size() - x, 0x4000);
f->chunk_crcs.emplace_back(::crc32(f->data->data() + x, chunk_bytes));
}
if (should_compute_crc32s) {
auto data = f->load_data(); // Sets f->size
f->crc32 = crc32(data->data(), f->size);
for (size_t x = 0; x < data->size(); x += 0x4000) {
size_t chunk_bytes = min<size_t>(f->size - x, 0x4000);
f->chunk_crcs.emplace_back(::crc32(data->data() + x, chunk_bytes));
}
vector<shared_ptr<JSONObject>> new_cache_item({
make_json_int(f->data->size()),
make_json_int(st.st_mtime),
make_json_int(f->crc32),
});
new_metadata_cache_json->as_dict().emplace(
relative_item_path, make_json_list(move(new_cache_item)));
// File was modified or cache item was missing; make a new cache item
vector<shared_ptr<JSONObject>> chunk_crcs_item;
for (uint32_t chunk_crc : f->chunk_crcs) {
chunk_crcs_item.emplace_back(make_json_int(chunk_crc));
}
vector<shared_ptr<JSONObject>> new_cache_item({
make_json_int(f->size),
make_json_int(st.st_mtime),
make_json_int(f->crc32),
make_json_list(move(chunk_crcs_item)),
});
new_metadata_cache_json->as_dict().emplace(
relative_item_path, make_json_list(move(new_cache_item)));
} else {
// File was not modified and cache item was valid; just use the
// existing cache item
new_metadata_cache_json->as_dict().emplace(
relative_item_path, cache_item_json);
}
this->files_by_patch_order.emplace_back(f);
this->files_by_name.emplace(relative_item_path, f);
patch_index_log.info("Added file %s (%zu bytes; %zu chunks; %08" PRIX32 ")",
full_item_path.c_str(), f->data->size(), f->chunk_crcs.size(), f->crc32);
patch_index_log.info("Added file %s (%" PRIu32 " bytes; %zu chunks; %08" PRIX32 ")",
full_item_path.c_str(), f->size, f->chunk_crcs.size(), f->crc32);
}
}
@@ -121,12 +154,12 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
}
}
const vector<shared_ptr<const PatchFileIndex::File>>&
const vector<shared_ptr<PatchFileIndex::File>>&
PatchFileIndex::all_files() const {
return this->files_by_patch_order;
}
shared_ptr<const PatchFileIndex::File> PatchFileIndex::get(
shared_ptr<PatchFileIndex::File> PatchFileIndex::get(
const string& filename) const {
return this->files_by_name.at(filename);
}
src/PatchFileIndex.hh
+12 -9
View File
@@ -14,35 +14,38 @@ struct PatchFileIndex {
explicit PatchFileIndex(const std::string& root_dir);
struct File {
PatchFileIndex* index;
std::vector<std::string> path_directories;
std::string name;
std::shared_ptr<const std::string> data;
std::shared_ptr<const std::string> loaded_data;
std::vector<uint32_t> chunk_crcs;
uint32_t crc32;
uint32_t size;
File();
explicit File(PatchFileIndex* index);
std::shared_ptr<const std::string> load_data();
};
const std::vector<std::shared_ptr<const File>>& all_files() const;
std::shared_ptr<const File> get(const std::string& filename) const;
const std::vector<std::shared_ptr<File>>& all_files() const;
std::shared_ptr<File> get(const std::string& filename) const;
private:
std::vector<std::shared_ptr<const File>> files_by_patch_order;
std::unordered_map<std::string, std::shared_ptr<const File>> files_by_name;
std::vector<std::shared_ptr<File>> files_by_patch_order;
std::unordered_map<std::string, std::shared_ptr<File>> files_by_name;
std::string root_dir;
};
struct PatchFileChecksumRequest {
std::shared_ptr<const PatchFileIndex::File> file;
std::shared_ptr<PatchFileIndex::File> file;
uint32_t crc32;
uint32_t size;
bool response_received;
explicit PatchFileChecksumRequest(std::shared_ptr<const PatchFileIndex::File> file)
explicit PatchFileChecksumRequest(std::shared_ptr<PatchFileIndex::File> file)
: file(file), crc32(0), size(0), response_received(false) { }
inline bool needs_update() const {
return !this->response_received ||
(this->crc32 != this->file->crc32) ||
(this->size != this->file->data->size());
(this->size != this->file->size);
}
};
src/ReceiveCommands.cc
+3 -3
View File
@@ -2621,9 +2621,9 @@ static void on_checksums_done_patch(shared_ptr<ServerState>,
throw runtime_error("client did not respond to checksum request");
}
if (req.needs_update()) {
c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %zu/%" PRIu32 ")",
req.file->name.c_str(), req.file->crc32, req.crc32, req.file->data->size(), req.size);
start_cmd.total_bytes += req.file->data->size();
c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %" PRIu32 "/%" PRIu32 ")",
req.file->name.c_str(), req.file->crc32, req.crc32, req.file->size, req.size);
start_cmd.total_bytes += req.file->size;
start_cmd.num_files++;
} else {
c->log.info("File %s is up to date", req.file->name.c_str());
src/SendCommands.cc
+5 -4
View File
@@ -471,8 +471,8 @@ void send_enter_directory_patch(shared_ptr<Client> c, const string& dir) {
send_command_t(c, 0x09, 0x00, cmd);
}
void send_patch_file(shared_ptr<Client> c, shared_ptr<const PatchFileIndex::File> f) {
S_OpenFile_Patch_06 open_cmd = {0, f->data->size(), f->name};
void send_patch_file(shared_ptr<Client> c, shared_ptr<PatchFileIndex::File> f) {
S_OpenFile_Patch_06 open_cmd = {0, f->size, f->name};
send_command_t(c, 0x06, 0x00, open_cmd);
for (size_t x = 0; x < f->chunk_crcs.size(); x++) {
@@ -480,11 +480,12 @@ void send_patch_file(shared_ptr<Client> c, shared_ptr<const PatchFileIndex::File
// Channel::send that takes iovecs or something to avoid these dumb massive
// string copies.
StringWriter w;
size_t chunk_size = min<uint32_t>(f->data->size() - x * 0x4000, 0x4000);
auto data = f->load_data();
size_t chunk_size = min<uint32_t>(f->size - (x * 0x4000), 0x4000);
S_WriteFileHeader_Patch_07 write_cmd_header = {
x, f->chunk_crcs[x], chunk_size};
w.put(write_cmd_header);
w.write(f->data->data() + x * 0x4000, chunk_size);
w.write(data->data() + (x * 0x4000), chunk_size);
while (w.size() & 7) {
w.put_u8(0);
}
src/SendCommands.hh
+1 -1
View File
@@ -142,7 +142,7 @@ void send_approve_player_choice_bb(std::shared_ptr<Client> c);
void send_complete_player_bb(std::shared_ptr<Client> c);
void send_enter_directory_patch(std::shared_ptr<Client> c, const std::string& dir);
void send_patch_file(std::shared_ptr<Client> c, std::shared_ptr<const PatchFileIndex::File> f);
void send_patch_file(std::shared_ptr<Client> c, std::shared_ptr<PatchFileIndex::File> f);
void send_message_box(std::shared_ptr<Client> c, const std::u16string& text);
void send_lobby_name(std::shared_ptr<Client> c, const std::u16string& text);
src/ServerState.cc
+1 -1
View File
@@ -436,7 +436,7 @@ shared_ptr<const string> ServerState::load_bb_file(
// First, look in the patch tree's data directory
string patch_index_path = "./data/" + patch_index_filename;
try {
auto ret = this->bb_patch_file_index->get(patch_index_path)->data;
auto ret = this->bb_patch_file_index->get(patch_index_path)->load_data();
static_game_data_log.info("Loaded %s from file in BB patch tree", patch_index_path.c_str());
return ret;
} catch (const out_of_range&) {