cache file checksums in patch trees

src/PatchFileIndex.cc

@@ -17,22 +17,25 @@ using namespace std;
 
 PatchFileIndex::File::File() : crc32(0) { }
 
-void PatchFileIndex::File::load_data(const string& root_dir) {
-  string full_path = root_dir + '/' + join(this->path_directories, "/") + '/' + this->name;
-  auto f = fopen_unique(full_path, "rb");
-
-  string file_data = load_file(full_path);
-  this->data.reset(new string(move(file_data)));
-  this->crc32 = ::crc32(this->data->data(), this->data->size());
-  for (size_t x = 0; x < this->data->size(); x += 0x4000) {
-    size_t chunk_bytes = min<size_t>(this->data->size() - x, 0x4000);
-    this->chunk_crcs.emplace_back(::crc32(this->data->data() + x, chunk_bytes));
-  }
-}
-
 PatchFileIndex::PatchFileIndex(const string& root_dir)
   : root_dir(root_dir) {
 
+  string metadata_cache_filename = root_dir + "/.metadata-cache.json";
+  shared_ptr<JSONObject> metadata_cache_json;
+  try {
+    string metadata_text = load_file(metadata_cache_filename);
+    metadata_cache_json = JSONObject::parse(metadata_text);
+    patch_index_log.info("Loaded patch metadata cache from %s", metadata_cache_filename.c_str());
+  } catch (const exception& e) {
+    metadata_cache_json = make_json_dict({});
+    patch_index_log.warning("Cannot load patch metadata cache from %s: %s", metadata_cache_filename.c_str(), e.what());
+  }
+
+  // Assuming it's rare for patch files to change, we skip writing the metadata
+  // cache if no files were changed at all (which should usually be the case)
+  bool should_write_metadata_cache = false;
+  shared_ptr<JSONObject> new_metadata_cache_json = make_json_dict({});
+
   vector<string> path_directories;
   function<void(const string&)> collect_dir = [&](const string& dir) -> void {
     path_directories.emplace_back(dir);
@@ -52,10 +55,48 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
       if (isdir(full_item_path)) {
         collect_dir(item);
       } else if (isfile(full_item_path)) {
+
+        auto st = stat(full_item_path);
+
         shared_ptr<File> f(new File());
         f->path_directories = path_directories;
         f->name = item;
-        f->load_data(root_dir);
+
+        int64_t file_crc32 = -1;
+        try {
+          auto cache_item = metadata_cache_json->at(relative_item_path)->as_list();
+          uint64_t cached_size = cache_item.at(0)->as_int();
+          uint64_t cached_mtime = cache_item.at(1)->as_int();
+          if (static_cast<uint64_t>(st.st_mtime) != cached_mtime) {
+            throw runtime_error("file has been modified");
+          }
+          if (static_cast<uint64_t>(st.st_size) != cached_size) {
+            throw runtime_error("file size has changed");
+          }
+          file_crc32 = cache_item.at(2)->as_int();
+          patch_index_log.info("Using cached checksum for %s", relative_item_path.c_str());
+        } catch (const exception& e) {
+          should_write_metadata_cache = true;
+          patch_index_log.info("Cannot use cached checksum for %s: %s", relative_item_path.c_str(), e.what());
+        }
+
+        string file_data = load_file(full_item_path);
+        f->data.reset(new string(move(file_data)));
+        f->crc32 = (file_crc32 < 0)
+            ? ::crc32(f->data->data(), f->data->size()) : file_crc32;
+        for (size_t x = 0; x < f->data->size(); x += 0x4000) {
+          size_t chunk_bytes = min<size_t>(f->data->size() - x, 0x4000);
+          f->chunk_crcs.emplace_back(::crc32(f->data->data() + x, chunk_bytes));
+        }
+
+        vector<shared_ptr<JSONObject>> new_cache_item({
+          make_json_int(f->data->size()),
+          make_json_int(st.st_mtime),
+          make_json_int(f->crc32),
+        });
+        new_metadata_cache_json->as_dict().emplace(
+            relative_item_path, make_json_list(move(new_cache_item)));
+
         this->files_by_patch_order.emplace_back(f);
         this->files_by_name.emplace(relative_item_path, f);
         patch_index_log.info("Added file %s (%zu bytes; %zu chunks; %08" PRIX32 ")",
@@ -67,6 +108,17 @@ PatchFileIndex::PatchFileIndex(const string& root_dir)
   };
 
   collect_dir(".");
+
+  if (should_write_metadata_cache) {
+    try {
+      save_file(metadata_cache_filename, new_metadata_cache_json->serialize());
+      patch_index_log.info("Saved patch metadata cache to %s", metadata_cache_filename.c_str());
+    } catch (const exception& e) {
+      patch_index_log.warning("Cannot save patch metadata cache to %s: %s", metadata_cache_filename.c_str(), e.what());
+    }
+  } else {
+    patch_index_log.info("No files were modified; skipping metadata cache update");
+  }
 }
 
 const vector<shared_ptr<const PatchFileIndex::File>>&