incentive/psopeeps-newserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

also store chunk crcs in patch metadata cache

Browse Source
This commit is contained in:
Martin Michelsen
2022-09-03 23:24:35 -07:00
parent f166dae1c6
commit 764a930213
9 changed files with 83 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Main.cc
+1 -1
View File
@@ -513,7 +513,7 @@ int main(int argc, char** argv) {
state->bb_patch_file_index.reset(new PatchFileIndex("system/patch-bb"));
try {
auto gsl_file = state->bb_patch_file_index->get("./data/data.gsl");
state->bb_data_gsl.reset(new GSLArchive(gsl_file->data));
state->bb_data_gsl.reset(new GSLArchive(gsl_file->load_data()));
config_log.info("data.gsl found in BB patch files");
} catch (const out_of_range&) {
config_log.info("data.gsl is not present in BB patch files");
src/PatchFileIndex.cc
+58 -25
View File
@@ -15,7 +15,21 @@ using namespace std;
PatchFileIndex::File::File() : crc32(0) { }
PatchFileIndex::File::File(PatchFileIndex* index)
: index(index), crc32(0), size(0) { }
std::shared_ptr<const std::string> PatchFileIndex::File::load_data() {
if (!this->loaded_data) {
string relative_path = join(this->path_directories, "/") + "/" + this->name;
string full_path = this->index->root_dir + "/" + relative_path;
patch_index_log.info("Loading data for %s", relative_path.c_str());
this->loaded_data.reset(new string(load_file(full_path)));
this->size = this->loaded_data->size();
}
return this->loaded_data;
}
PatchFileIndex::PatchFileIndex(const string& root_dir)
: root_dir(root_dir) {
@@ -58,12 +72,14 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
auto st = stat(full_item_path);
shared_ptr<File> f(new File());
shared_ptr<File> f(new File(this));
f->path_directories = path_directories;
f->name = item;
int64_t file_crc32 = -1;
bool should_compute_crc32s = true;
shared_ptr<JSONObject> cache_item_json;
try {
cache_item_json = metadata_cache_json->at(relative_item_path);
auto cache_item = metadata_cache_json->at(relative_item_path)->as_list();
uint64_t cached_size = cache_item.at(0)->as_int();
uint64_t cached_mtime = cache_item.at(1)->as_int();
@@ -73,34 +89,51 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
if (static_cast<uint64_t>(st.st_size) != cached_size) {
throw runtime_error("file size has changed");
}
file_crc32 = cache_item.at(2)->as_int();
patch_index_log.info("Using cached checksum for %s", relative_item_path.c_str());
f->size = cached_size;
f->crc32 = cache_item.at(2)->as_int();
for (const auto& chunk_crc32_json : cache_item.at(3)->as_list()) {
f->chunk_crcs.emplace_back(chunk_crc32_json->as_int());
}
should_compute_crc32s = false;
patch_index_log.info("Using cached checksums for %s", relative_item_path.c_str());
} catch (const exception& e) {
should_write_metadata_cache = true;
patch_index_log.info("Cannot use cached checksum for %s: %s", relative_item_path.c_str(), e.what());
patch_index_log.info("Cannot use cached checksums for %s: %s", relative_item_path.c_str(), e.what());
}
string file_data = load_file(full_item_path);
f->data.reset(new string(move(file_data)));
f->crc32 = (file_crc32 < 0)
? ::crc32(f->data->data(), f->data->size()) : file_crc32;
for (size_t x = 0; x < f->data->size(); x += 0x4000) {
size_t chunk_bytes = min<size_t>(f->data->size() - x, 0x4000);
f->chunk_crcs.emplace_back(::crc32(f->data->data() + x, chunk_bytes));
}
if (should_compute_crc32s) {
auto data = f->load_data(); // Sets f->size
f->crc32 = crc32(data->data(), f->size);
for (size_t x = 0; x < data->size(); x += 0x4000) {
size_t chunk_bytes = min<size_t>(f->size - x, 0x4000);
f->chunk_crcs.emplace_back(::crc32(data->data() + x, chunk_bytes));
}
vector<shared_ptr<JSONObject>> new_cache_item({
make_json_int(f->data->size()),
make_json_int(st.st_mtime),
make_json_int(f->crc32),
});
new_metadata_cache_json->as_dict().emplace(
relative_item_path, make_json_list(move(new_cache_item)));
// File was modified or cache item was missing; make a new cache item
vector<shared_ptr<JSONObject>> chunk_crcs_item;
for (uint32_t chunk_crc : f->chunk_crcs) {
chunk_crcs_item.emplace_back(make_json_int(chunk_crc));
}
vector<shared_ptr<JSONObject>> new_cache_item({
make_json_int(f->size),
make_json_int(st.st_mtime),
make_json_int(f->crc32),
make_json_list(move(chunk_crcs_item)),
});
new_metadata_cache_json->as_dict().emplace(
relative_item_path, make_json_list(move(new_cache_item)));
} else {
// File was not modified and cache item was valid; just use the
// existing cache item
new_metadata_cache_json->as_dict().emplace(
relative_item_path, cache_item_json);
}
this->files_by_patch_order.emplace_back(f);
this->files_by_name.emplace(relative_item_path, f);
patch_index_log.info("Added file %s (%zu bytes; %zu chunks; %08" PRIX32 ")",
full_item_path.c_str(), f->data->size(), f->chunk_crcs.size(), f->crc32);
patch_index_log.info("Added file %s (%" PRIu32 " bytes; %zu chunks; %08" PRIX32 ")",
full_item_path.c_str(), f->size, f->chunk_crcs.size(), f->crc32);
}
}
@@ -121,12 +154,12 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
}
}
const vector<shared_ptr<const PatchFileIndex::File>>&
const vector<shared_ptr<PatchFileIndex::File>>&
PatchFileIndex::all_files() const {
return this->files_by_patch_order;
}
shared_ptr<const PatchFileIndex::File> PatchFileIndex::get(
shared_ptr<PatchFileIndex::File> PatchFileIndex::get(
const string& filename) const {
return this->files_by_name.at(filename);
}
src/PatchFileIndex.hh
+12 -9
View File
@@ -14,35 +14,38 @@ struct PatchFileIndex {
explicit PatchFileIndex(const std::string& root_dir);
struct File {
PatchFileIndex* index;
std::vector<std::string> path_directories;
std::string name;
std::shared_ptr<const std::string> data;
std::shared_ptr<const std::string> loaded_data;
std::vector<uint32_t> chunk_crcs;
uint32_t crc32;
uint32_t size;
File();
explicit File(PatchFileIndex* index);
std::shared_ptr<const std::string> load_data();
};
const std::vector<std::shared_ptr<const File>>& all_files() const;
std::shared_ptr<const File> get(const std::string& filename) const;
const std::vector<std::shared_ptr<File>>& all_files() const;
std::shared_ptr<File> get(const std::string& filename) const;
private:
std::vector<std::shared_ptr<const File>> files_by_patch_order;
std::unordered_map<std::string, std::shared_ptr<const File>> files_by_name;
std::vector<std::shared_ptr<File>> files_by_patch_order;
std::unordered_map<std::string, std::shared_ptr<File>> files_by_name;
std::string root_dir;
};
struct PatchFileChecksumRequest {
std::shared_ptr<const PatchFileIndex::File> file;
std::shared_ptr<PatchFileIndex::File> file;
uint32_t crc32;
uint32_t size;
bool response_received;
explicit PatchFileChecksumRequest(std::shared_ptr<const PatchFileIndex::File> file)
explicit PatchFileChecksumRequest(std::shared_ptr<PatchFileIndex::File> file)
: file(file), crc32(0), size(0), response_received(false) { }
inline bool needs_update() const {
return !this->response_received ||
(this->crc32 != this->file->crc32) ||
(this->size != this->file->data->size());
(this->size != this->file->size);
}
};
src/ReceiveCommands.cc
+3 -3
View File
@@ -2621,9 +2621,9 @@ static void on_checksums_done_patch(shared_ptr<ServerState>,
throw runtime_error("client did not respond to checksum request");
}
if (req.needs_update()) {
c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %zu/%" PRIu32 ")",
req.file->name.c_str(), req.file->crc32, req.crc32, req.file->data->size(), req.size);
start_cmd.total_bytes += req.file->data->size();
c->log.info("File %s needs update (CRC: %08" PRIX32 "/%08" PRIX32 ", size: %" PRIu32 "/%" PRIu32 ")",
req.file->name.c_str(), req.file->crc32, req.crc32, req.file->size, req.size);
start_cmd.total_bytes += req.file->size;
start_cmd.num_files++;
} else {
c->log.info("File %s is up to date", req.file->name.c_str());
src/SendCommands.cc
+5 -4
View File
@@ -471,8 +471,8 @@ void send_enter_directory_patch(shared_ptr<Client> c, const string& dir) {
send_command_t(c, 0x09, 0x00, cmd);
}
void send_patch_file(shared_ptr<Client> c, shared_ptr<const PatchFileIndex::File> f) {
S_OpenFile_Patch_06 open_cmd = {0, f->data->size(), f->name};
void send_patch_file(shared_ptr<Client> c, shared_ptr<PatchFileIndex::File> f) {
S_OpenFile_Patch_06 open_cmd = {0, f->size, f->name};
send_command_t(c, 0x06, 0x00, open_cmd);
for (size_t x = 0; x < f->chunk_crcs.size(); x++) {
@@ -480,11 +480,12 @@ void send_patch_file(shared_ptr<Client> c, shared_ptr<const PatchFileIndex::File
// Channel::send that takes iovecs or something to avoid these dumb massive
// string copies.
StringWriter w;
size_t chunk_size = min<uint32_t>(f->data->size() - x * 0x4000, 0x4000);
auto data = f->load_data();
size_t chunk_size = min<uint32_t>(f->size - (x * 0x4000), 0x4000);
S_WriteFileHeader_Patch_07 write_cmd_header = {
x, f->chunk_crcs[x], chunk_size};
w.put(write_cmd_header);
w.write(f->data->data() + x * 0x4000, chunk_size);
w.write(data->data() + (x * 0x4000), chunk_size);
while (w.size() & 7) {
w.put_u8(0);
}
src/SendCommands.hh
+1 -1
View File
@@ -142,7 +142,7 @@ void send_approve_player_choice_bb(std::shared_ptr<Client> c);
void send_complete_player_bb(std::shared_ptr<Client> c);
void send_enter_directory_patch(std::shared_ptr<Client> c, const std::string& dir);
void send_patch_file(std::shared_ptr<Client> c, std::shared_ptr<const PatchFileIndex::File> f);
void send_patch_file(std::shared_ptr<Client> c, std::shared_ptr<PatchFileIndex::File> f);
void send_message_box(std::shared_ptr<Client> c, const std::u16string& text);
void send_lobby_name(std::shared_ptr<Client> c, const std::u16string& text);
src/ServerState.cc
+1 -1
View File
@@ -436,7 +436,7 @@ shared_ptr<const string> ServerState::load_bb_file(
// First, look in the patch tree's data directory
string patch_index_path = "./data/" + patch_index_filename;
try {
auto ret = this->bb_patch_file_index->get(patch_index_path)->data;
auto ret = this->bb_patch_file_index->get(patch_index_path)->load_data();
static_game_data_log.info("Loaded %s from file in BB patch tree", patch_index_path.c_str());
return ret;
} catch (const out_of_range&) {