describe DC save file formats; add decrypt/encrypt actions

src/ChatCommands.cc

@@ -1728,8 +1728,8 @@ static void server_command_loadchar(shared_ptr<Client> c, const std::string& arg
     };
 
     if (c->version() == Version::DC_V2) {
-      auto dc_char = make_shared<PSODCV2CharacterFile>(c->character()->to_dc_v2());
+      auto dc_char = make_shared<PSODCV2CharacterFile::Character>(c->character()->to_dc_v2());
-      send_set_extended_player_info.operator()<PSODCV2CharacterFile>(c, dc_char);
+      send_set_extended_player_info.operator()<PSODCV2CharacterFile::Character>(c, dc_char);
     } else if (c->version() == Version::GC_NTE) {
       auto gc_char = make_shared<PSOGCNTECharacterFileCharacter>(c->character()->to_gc_nte());
       send_set_extended_player_info.operator()<PSOGCNTECharacterFileCharacter>(c, gc_char);

src/DCSerialNumbers.cc

@@ -2,6 +2,7 @@
 
 #include <stdint.h>
 
+#include <mutex>
 #include <phosg/Random.hh>
 #include <phosg/Strings.hh>
 #include <phosg/Time.hh>
@@ -24,6 +25,7 @@ static const uint32_t primes1[] = {
     0x313, 0x31D, 0x329, 0x32B, 0x335, 0x337, 0x33B, 0x33D, 0x347, 0x355, 0x359,
     0x35B, 0x35F, 0x36D, 0x371, 0x373, 0x377, 0x38B, 0x38F, 0x397, 0x3A1, 0x3A9,
     0x3AD, 0x3B3, 0x3B9, 0x3C7, 0x3CB, 0x3D1, 0x3D7, 0x3DF, 0x3E5};
+
 static const uint32_t primes2[] = {
     0x3F1, 0x3F5, 0x3FB, 0x3FD, 0x407, 0x409, 0x40F, 0x419, 0x41B, 0x425, 0x427,
     0x42D, 0x43F, 0x443, 0x445, 0x449, 0x44F, 0x455, 0x45D, 0x463, 0x469, 0x47F,
@@ -135,6 +137,8 @@ static const uint32_t primes2[] = {
     0x2627, 0x2629, 0x2635, 0x263B, 0x263F, 0x264B, 0x2653, 0x2659, 0x2665,
     0x2669, 0x266F, 0x267B, 0x2681, 0x2683, 0x268F, 0x269B, 0x269F, 0x26AD,
     0x26B3, 0x26C3, 0x26C9, 0x26CB, 0x26D5, 0x26DD, 0x26EF, 0x26F5};
+static constexpr size_t num_primes2 = sizeof(primes2) / sizeof(primes2[0]);
+
 static const uint32_t primes3[] = {
     0x2717, 0x2719, 0x2735, 0x2737, 0x274D, 0x2753, 0x2755, 0x275F, 0x276B,
     0x276D, 0x2773, 0x2777, 0x277F, 0x2795, 0x279B, 0x279D, 0x27A7, 0x27AF,
@@ -1107,17 +1111,21 @@ static const uint32_t primes3[] = {
     0x18569, 0x1857B, 0x1857D, 0x18581, 0x18587, 0x18589, 0x18595, 0x185B1,
     0x185B7, 0x185CB, 0x185D1, 0x185E1, 0x185E9, 0x185EF, 0x185F5, 0x185F9,
     0x185FF, 0x18613, 0x1861F};
+static constexpr size_t num_primes3 = sizeof(primes3) / sizeof(primes3[0]);
 
 static bool check_prime3(uint64_t prime3) {
   static vector<bool> primes3_set;
+  static mutex primes3_init_mutex;
+  if (primes3_set.empty()) {
+    lock_guard g(primes3_init_mutex);
     if (primes3_set.empty()) {
-    size_t num_primes3 = sizeof(primes3) / sizeof(primes3[0]);
       size_t primes3_set_size = primes3[num_primes3 - 1] - primes3[0] + 1;
       primes3_set.resize(primes3_set_size, false);
       for (size_t z = 0; z < num_primes3; z++) {
         primes3_set[primes3[z] - primes3[0]] = true;
       }
     }
+  }
   uint64_t offset = prime3 - primes3[0];
   if (offset >= primes3_set.size()) {
     return false;
@@ -1227,8 +1235,8 @@ bool dc_serial_number_is_valid_slow(const string& s, uint8_t domain, uint8_t sub
   }
 
   for (; offset1 < limit1; offset1++) {
-    for (size_t offset2 = 0; offset2 < sizeof(primes2) / sizeof(primes2[0]); offset2++) {
+    for (size_t offset2 = 0; offset2 < num_primes2; offset2++) {
-      for (size_t offset3 = 0; offset3 < sizeof(primes3) / sizeof(primes3[0]); offset3++) {
+      for (size_t offset3 = 0; offset3 < num_primes3; offset3++) {
         if (primes1[offset1] * primes2[offset2] * primes3[offset3] == serial_number) {
           return true;
         }
@@ -1251,7 +1259,7 @@ bool decoded_dc_serial_number_is_valid_fast(uint32_t serial_number, uint8_t doma
         continue;
       }
       uint64_t sub1 = sub0 / primes1[offset1];
-      for (size_t offset2 = 0; offset2 < sizeof(primes2) / sizeof(primes2[0]); offset2++) {
+      for (size_t offset2 = 0; offset2 < num_primes2; offset2++) {
         if (sub1 % primes2[offset2]) {
           continue;
         }
@@ -1293,8 +1301,8 @@ string generate_dc_serial_number(uint8_t domain, uint8_t subdomain) {
 
   size_t det1 = (subdomain == 0xFF) ? phosg::random_object<uint32_t>() : subdomain;
   size_t index1 = offset1 + (det1 % (limit1 - offset1));
-  size_t index2 = phosg::random_object<uint32_t>() % (sizeof(primes2) / sizeof(primes2[0]));
+  size_t index2 = phosg::random_object<uint32_t>() % num_primes2;
-  size_t index3 = phosg::random_object<uint32_t>() % (sizeof(primes3) / sizeof(primes3[0]));
+  size_t index3 = phosg::random_object<uint32_t>() % num_primes3;
   uint32_t value = primes1[index1] * primes2[index2] * primes3[index3];
   string s = phosg::string_printf("%08X", value);
 
@@ -1306,54 +1314,79 @@ string generate_dc_serial_number(uint8_t domain, uint8_t subdomain) {
 }
 
 unordered_map<uint32_t, string> generate_all_dc_serial_numbers(uint8_t domain, uint8_t subdomain) {
-  vector<uint8_t> domains;
+  DCSerialNumberIterator iter;
-  if (domain == 0xFF) {
-    domains.emplace_back(0x00);
-    domains.emplace_back(0x01);
-    domains.emplace_back(0x02);
-  } else {
-    domains.emplace_back(domain);
-  }
 
-  vector<uint8_t> subdomains;
+  if (domain < 3) {
-  if (subdomain == 0xFF) {
+    iter.domain = domain;
-    subdomains.emplace_back(0x00);
+    iter.end_domain = domain + 1;
-    subdomains.emplace_back(0x01);
+  } else if (domain != 0xFF) {
-    subdomains.emplace_back(0x02);
-  } else {
-    subdomains.emplace_back(subdomain);
-  }
-
-  unordered_map<uint32_t, string> ret;
-  for (uint8_t domain : domains) {
-    size_t offset1, limit1;
-    if (domain == 0) {
-      offset1 = 0x00;
-      limit1 = 0x03;
-    } else if (domain == 1) {
-      offset1 = 0x1E;
-      limit1 = 0x21;
-    } else if (domain == 2) {
-      offset1 = 0x3C;
-      limit1 = 0x3F;
-    } else {
     throw runtime_error("invalid domain");
   }
 
-    for (uint8_t subdomain : subdomains) {
+  if (subdomain < 3) {
-      size_t index1 = offset1 + (subdomain % (limit1 - offset1));
+    iter.subdomain = subdomain;
-      for (size_t index2 = 0; index2 < sizeof(primes2) / sizeof(primes2[0]); index2++) {
+    iter.start_subdomain = subdomain;
-        for (size_t index3 = 0; index3 < sizeof(primes3) / sizeof(primes3[0]); index3++) {
+    iter.end_subdomain = subdomain + 1;
-          uint32_t value = primes1[index1] * primes2[index2] * primes3[index3];
+  } else if (subdomain != 0xFF) {
-          ret[encode_dc_serial_number_int(value)].push_back(((domain << 2) & 3) | (subdomain & 3));
+    throw runtime_error("invalid subdomain");
-        }
-        fprintf(stderr, "... domain=%hhu subdomain=%hhu index2=%zu results=%zu (0x%zX)\n", domain, subdomain, index2, ret.size(), ret.size());
   }
+
+  uint32_t serial_number;
+  unordered_map<uint32_t, string> ret;
+  while ((serial_number = iter.next()) != 0) {
+    ret[serial_number].push_back(((iter.domain << 2) & 3) | (iter.subdomain & 3));
+    if (iter.index3 == 0) {
+      fprintf(stderr, "... (it) domain=%hhu subdomain=%hhu index2=%hu results=%zu (0x%zX)\n", iter.domain, iter.subdomain, iter.index2, ret.size(), ret.size());
     }
   }
   return ret;
 }
 
+uint32_t DCSerialNumberIterator::next() {
+  if (!this->started) {
+    this->started = true;
+  } else if (!this->complete) {
+    this->index3++;
+    if (this->index3 >= num_primes3) {
+      this->index3 = 0;
+      this->index2++;
+      if (this->index2 >= num_primes2) {
+        this->index2 = 0;
+        this->subdomain++;
+        if (this->subdomain >= this->end_subdomain) {
+          this->subdomain = this->start_subdomain;
+          this->domain++;
+          if (this->domain >= this->end_domain) {
+            this->serial_number = 0;
+            this->complete = true;
+          }
+        }
+      }
+    }
+  }
+  if (!this->complete) {
+    size_t index1 = (30 * this->domain) + (this->subdomain % 3);
+    return encode_dc_serial_number_int(primes1[index1] * primes2[this->index2] * primes3[this->index3]);
+  } else {
+    return 0;
+  }
+}
+
+size_t DCSerialNumberIterator::total_count() const {
+  return (this->end_domain - this->start_domain) * (this->end_subdomain - this->start_subdomain) * num_primes2 * num_primes3;
+}
+
+size_t DCSerialNumberIterator::progress() const {
+  size_t domains_done = this->domain - this->start_domain;
+  size_t subdomains_per_domain = this->end_subdomain - this->start_subdomain;
+  size_t subdomains_done = this->subdomain - this->start_subdomain;
+  return (
+      (domains_done * subdomains_per_domain * num_primes2 * num_primes3) +
+      (subdomains_done * num_primes2 * num_primes3) +
+      (this->index2 * num_primes3) +
+      this->index3);
+}
+
 void dc_serial_number_speed_test(uint64_t seed) {
   uint32_t effective_seed = (seed & 0xFFFFFFFF00000000) ? phosg::random_object<uint32_t>() : seed;
   fprintf(stderr, "Product speed test with seed=%08" PRIX32 "\n", effective_seed);

src/DCSerialNumbers.hh

@@ -20,6 +20,25 @@ bool decoded_dc_serial_number_is_valid_fast(
 std::string generate_dc_serial_number(uint8_t domain, uint8_t subdomain = 0xFF);
 std::unordered_map<uint32_t, std::string> generate_all_dc_serial_numbers(uint8_t domain = 0xFF, uint8_t subdomain = 0xFF);
 
+struct DCSerialNumberIterator {
+  bool started = false;
+  bool complete = false;
+  uint8_t domain = 0;
+  uint8_t start_domain = 0;
+  uint8_t end_domain = 3;
+  uint8_t subdomain = 0;
+  uint8_t start_subdomain = 0;
+  uint8_t end_subdomain = 3;
+  uint16_t index2 = 0;
+  uint16_t index3 = 0;
+  uint32_t serial_number = 0;
+
+  uint32_t next();
+
+  size_t total_count() const;
+  size_t progress() const;
+};
+
 void dc_serial_number_speed_test(uint64_t seed = 0xFFFFFFFFFFFFFFFF);
 
 struct EncryptedDCv2Executables {

src/Main.cc

@@ -610,8 +610,6 @@ Action a_parse_pc_v2_registry(
           serial_number, serial_number, access_data.c_str(), email_data.c_str());
     });
 
-// ./newserv generate-pc-v2-registry --serial-number=386248990 --access-key=14575600 --email=mjem@wildblue.net
-
 Action a_generate_pc_v2_registry(
     "generate-pc-v2-registry", "\
   generate-pc-v2-registry <OPTIONS> [OUTPUT-FILENAME]\n\
@@ -674,6 +672,172 @@ Action a_decrypt_challenge_data(
       write_output_data(args, data.data(), data.size(), "dec");
     });
 
+static void a_encrypt_decrypt_vms_save_fn(phosg::Arguments& args) {
+  bool is_decrypt = (args.get<string>(0) == "decrypt-vms-save");
+  bool skip_checksum = args.get<bool>("skip-checksum");
+  string serial_number_str = args.get<string>("serial-number");
+  int64_t override_round2_seed = args.get<int64_t>("round2-seed", -1, phosg::Arguments::IntFormat::HEX);
+
+  int64_t round1_seed = serial_number_str.empty() ? -1 : stoul(serial_number_str, nullptr, 16);
+
+  auto data = read_input_data(args);
+  phosg::StringReader r(data);
+  const auto& header = r.get<PSOVMSFileHeader>();
+  header.check();
+  r.skip(header.icon_data_size());
+
+  size_t data_start_offset = r.where();
+
+  auto process_file = [&]<typename StructT, bool UseIterator, size_t ChecksumLength = sizeof(StructT)>() {
+    if (is_decrypt) {
+      const void* data_section = r.getv(header.data_size);
+      if (round1_seed < 0) {
+        size_t num_threads = args.get<size_t>("threads", 0);
+        if (num_threads == 0) {
+          num_threads = thread::hardware_concurrency();
+        }
+
+        mutex output_lock;
+        if (UseIterator) {
+          DCSerialNumberIterator iter;
+          mutex iter_lock;
+          atomic<bool> seed_found = false;
+          auto thread_fn = [&]() -> void {
+            for (;;) {
+              uint32_t serial_number;
+              {
+                lock_guard g(iter_lock);
+                serial_number = iter.next();
+              }
+              if (serial_number == 0) {
+                return;
+              }
+              try {
+                auto decrypted = decrypt_fixed_size_data_section_t<StructT, false, ChecksumLength>(
+                    data_section, sizeof(StructT), serial_number, skip_checksum, override_round2_seed);
+
+                seed_found = true;
+                {
+                  lock_guard g(iter_lock);
+                  iter.complete = true;
+                }
+                lock_guard g(output_lock);
+                fprintf(stderr, "\nFound serial number: %08" PRIX32 "\n", serial_number);
+                *reinterpret_cast<StructT*>(data.data() + data_start_offset) = decrypted;
+
+              } catch (const runtime_error&) {
+              }
+            }
+          };
+
+          vector<thread> threads;
+          while (threads.size() < num_threads) {
+            threads.emplace_back(thread_fn);
+          }
+          for (;;) {
+            usleep(1000000);
+            lock_guard g(iter_lock);
+            size_t progress = iter.progress();
+            size_t total_count = iter.total_count();
+            float progress_percent = static_cast<float>(progress * 100) / total_count;
+            fprintf(stderr, "... %zu/%zu (%g%%, domain %02hhX, subdomain %02hhX, index2 %04hX, index3 %04hX)\r",
+                progress, total_count, progress_percent, iter.domain, iter.subdomain, iter.index2, iter.index3);
+            if (iter.complete) {
+              break;
+            }
+          }
+          for (auto& th : threads) {
+            th.join();
+          }
+          if (!seed_found) {
+            throw runtime_error("no seed found");
+          }
+
+        } else {
+          uint64_t seed = phosg::parallel_range_blocks<uint64_t>([&](uint64_t serial_number, size_t) -> bool {
+            try {
+              auto decrypted = decrypt_fixed_size_data_section_t<StructT, false, ChecksumLength>(
+                  data_section, sizeof(StructT), serial_number, skip_checksum, override_round2_seed);
+
+              lock_guard g(output_lock);
+              fprintf(stderr, "\nFound serial number: %08" PRIX64 "\n", serial_number);
+              *reinterpret_cast<StructT*>(data.data() + data_start_offset) = decrypted;
+              return true;
+
+            } catch (const runtime_error&) {
+              return false;
+            }
+          },
+              0, 0x100000000, 0x1000, num_threads);
+          if (seed >= 0x100000000) {
+            throw runtime_error("no seed found");
+          }
+        }
+
+      } else {
+        auto decrypted = decrypt_fixed_size_data_section_t<StructT, false, ChecksumLength>(
+            data_section, sizeof(StructT), round1_seed, skip_checksum, override_round2_seed);
+        *reinterpret_cast<StructT*>(data.data() + data_start_offset) = decrypted;
+      }
+
+    } else {
+      const auto& s = r.get<StructT>();
+      auto encrypted = encrypt_fixed_size_data_section_t<StructT, false, ChecksumLength>(s, round1_seed);
+      if (data_start_offset + encrypted.size() > data.size()) {
+        throw runtime_error("encrypted result exceeds file size");
+      }
+      memcpy(data.data() + data_start_offset, encrypted.data(), encrypted.size());
+    }
+  };
+
+  bool is_v2 = header.is_v2();
+  if (!is_v2 && (header.data_size == sizeof(PSODCNTECharacterFile))) {
+    fprintf(stderr, "File type: DC NTE character\n");
+    process_file.template operator()<PSODCNTECharacterFile, false>();
+  } else if (!is_v2 && (header.data_size == sizeof(PSODCNTEGuildCardFile))) {
+    fprintf(stderr, "File type: DC NTE Guild Card list\n");
+    throw runtime_error("DC NTE Guild Card files are not encrypted");
+  } else if (!is_v2 && (header.data_size == sizeof(PSODC112000CharacterFile))) {
+    fprintf(stderr, "File type: DC 11/2000 character\n");
+    process_file.template operator()<PSODC112000CharacterFile, false>();
+  } else if (!is_v2 && (header.data_size == sizeof(PSODC112000GuildCardFile))) {
+    fprintf(stderr, "File type: DC 11/2000 Guild Card list\n");
+    throw runtime_error("DC 11/2000 Guild Card files are not encrypted");
+  } else if (!is_v2 && (header.data_size == sizeof(PSODCV1CharacterFile))) {
+    fprintf(stderr, "File type: DC v1 character\n");
+    process_file.template operator()<PSODCV1CharacterFile, true>();
+  } else if (is_v2 && (header.data_size == sizeof(PSODCV2CharacterFile))) {
+    fprintf(stderr, "File type: DC v2 character\n");
+    process_file.template operator()<PSODCV2CharacterFile, true>();
+  } else if (header.data_size == sizeof(PSODCV1V2GuildCardFile)) {
+    // There appears to be a copy/paste error here: the game uses the character
+    // file size when checksumming the Guild Card file, so we must do the same
+    if (!is_v2) {
+      fprintf(stderr, "File type: DC v1 Guild Card list\n");
+      static_assert(sizeof(PSODCV1CharacterFile) <= sizeof(PSODCV1V2GuildCardFile::EncryptedSection));
+      process_file.template operator()<PSODCV1V2GuildCardFile::EncryptedSection, true, sizeof(PSODCV1CharacterFile)>();
+    } else {
+      fprintf(stderr, "File type: DC v2 Guild Card list\n");
+      static_assert(sizeof(PSODCV2CharacterFile) <= sizeof(PSODCV1V2GuildCardFile::EncryptedSection));
+      process_file.template operator()<PSODCV1V2GuildCardFile::EncryptedSection, true, sizeof(PSODCV2CharacterFile)>();
+    }
+  } else {
+    throw runtime_error("unrecognized save type");
+  }
+
+  write_output_data(args, data.data(), data.size(), is_decrypt ? "vmsd" : "vms");
+}
+
+Action a_decrypt_vms_save("decrypt-vms-save", nullptr, a_encrypt_decrypt_vms_save_fn);
+Action a_encrypt_vms_save("encrypt-vms-save", "\
+  encrypt-gci-save --seed=SEED INPUT-FILENAME [OUTPUT-FILENAME]\n\
+  decrypt-gci-save [--seed=SEED] INPUT-FILENAME [OUTPUT-FILENAME]\n\
+    Encrypt or decrypt a character or Guild Card file in VMS format. If\n\
+    encrypting, the checksum is also recomputed and stored in the encrypted\n\
+    file. --seed is the encryption seed (serial number) specified as a 32-bit\n\
+    hexadecimal value.\n",
+    a_encrypt_decrypt_vms_save_fn);
+
 static void a_encrypt_decrypt_gci_save_fn(phosg::Arguments& args) {
   bool is_decrypt = (args.get<string>(0) == "decrypt-gci-save");
   bool skip_checksum = args.get<bool>("skip-checksum");
@@ -996,7 +1160,7 @@ Action a_salvage_gci(
       auto process_file = [&]<typename StructT>() {
         vector<multimap<size_t, uint32_t>> top_seeds_by_thread(
             num_threads ? num_threads : thread::hardware_concurrency());
-        phosg::parallel_range<uint64_t>(
+        phosg::parallel_range_blocks<uint64_t>(
             [&](uint64_t seed, size_t thread_num) -> bool {
               size_t zero_count;
               if (round2) {
@@ -1026,9 +1190,7 @@ Action a_salvage_gci(
               }
               return false;
             },
-            0,
+            0, 0x100000000, 0x1000, num_threads);
-            0x100000000,
-            num_threads);
 
         multimap<size_t, uint32_t> top_seeds;
         for (const auto& thread_top_seeds : top_seeds_by_thread) {
@@ -1108,7 +1270,7 @@ Action a_find_decryption_seed(
         return true;
       };
 
-      uint64_t seed = phosg::parallel_range<uint64_t>([&](uint64_t seed, size_t) -> bool {
+      uint64_t seed = phosg::parallel_range_blocks<uint64_t>([&](uint64_t seed, size_t) -> bool {
         string be_decrypt_buf = ciphertext.substr(0, max_plaintext_size);
         string le_decrypt_buf = ciphertext.substr(0, max_plaintext_size);
         if (uses_v3_encryption(version)) {
@@ -1137,7 +1299,7 @@ Action a_find_decryption_seed(
         }
         return false;
       },
-          0, 0x100000000, num_threads);
+          0, 0x100000000, 0x1000, num_threads);
 
       if (seed < 0x100000000) {
         phosg::log_info("Found seed %08" PRIX64, seed);
@@ -2500,7 +2662,7 @@ Action a_find_rare_enemy_seeds(
         return false;
       };
 
-      phosg::parallel_range<uint64_t>(thread_fn, 0, 0x100000000, num_threads, nullptr);
+      phosg::parallel_range_blocks<uint64_t>(thread_fn, 0, 0x100000000, 0x1000, num_threads, nullptr);
     });
 
 Action a_load_maps_test(
@@ -2639,45 +2801,65 @@ Action a_generate_dc_serial_number(
       fprintf(stdout, "%s\n", serial_number.c_str());
     });
 Action a_generate_all_dc_serial_numbers(
-    "generate-all-dc-serial-numbers", "\
+    "dc-serial-number-generator-test", nullptr,
-  generate-all-dc-serial-numbers\n\
-    Generate all possible PSO DC serial numbers.\n",
     +[](phosg::Arguments& args) {
       size_t num_threads = args.get<size_t>("threads", 0);
 
-      auto serial_numbers = generate_all_dc_serial_numbers();
+      vector<unordered_set<uint32_t>> serial_numbers;
-      fprintf(stdout, "%zu (0x%zX) serial numbers found\n", serial_numbers.size(), serial_numbers.size());
+      serial_numbers.resize(9);
-      for (const auto& it : serial_numbers) {
+
-        fprintf(stdout, "Valid serial number: %08" PRIX32, it.first);
+      DCSerialNumberIterator iter;
-        for (uint8_t where : it.second) {
+      uint32_t serial_number;
-          fprintf(stdout, " (domain=%hhu, subdomain=%hhu)",
+      size_t num_serial_numbers = 0;
-              static_cast<uint8_t>((where >> 2) & 3),
+      while ((serial_number = iter.next()) != 0) {
-              static_cast<uint8_t>(where & 3));
+        serial_numbers[iter.domain * 3 + iter.subdomain].emplace(serial_number);
+        if (((++num_serial_numbers) % 0x10000) == 0) {
+          fprintf(stderr, "... %08zX (domain=%02hhX, subdomain=%02hhX, index2=%04hX, index3=%04hX) counts=[%zu, %zu, %zu, %zu, %zu, %zu, %zu, %zu, %zu]\n",
+              num_serial_numbers, iter.domain, iter.subdomain, iter.index2, iter.index3,
+              serial_numbers[0].size(), serial_numbers[1].size(), serial_numbers[2].size(),
+              serial_numbers[3].size(), serial_numbers[4].size(), serial_numbers[5].size(),
+              serial_numbers[6].size(), serial_numbers[7].size(), serial_numbers[8].size());
         }
-        fputc('\n', stdout);
       }
 
-      atomic<uint64_t> num_valid_serial_numbers = 0;
+      array<atomic<size_t>, 9> found_counts = {0, 0, 0, 0, 0, 0, 0, 0, 0};
+      atomic<uint64_t> num_mismatches = 0;
       mutex output_lock;
       auto thread_fn = [&](uint64_t serial_number, size_t) -> bool {
         for (uint8_t domain = 0; domain < 3; domain++) {
           for (uint8_t subdomain = 0; subdomain < 3; subdomain++) {
-            if (dc_serial_number_is_valid_fast(serial_number, domain, subdomain)) {
+            bool is_valid = dc_serial_number_is_valid_fast(serial_number, domain, subdomain);
-              num_valid_serial_numbers++;
+            bool was_iterated = serial_numbers[domain * 3 + subdomain].count(serial_number);
+            if (is_valid != was_iterated) {
               lock_guard g(output_lock);
-              fprintf(stdout, "Valid serial number: %08" PRIX64 " (domain=%hhu, subdomain=%hhu)\n", serial_number, domain, subdomain);
+              fprintf(stdout, "Mismatch at %08" PRIX64 " (domain=%hhu, subdomain=%hhu): is_valid=%s, was_iterated=%s\n",
+                  serial_number, domain, subdomain, is_valid ? "true" : "false", was_iterated ? "true" : "false");
+            } else if (is_valid && was_iterated) {
+              found_counts[domain * 3 + subdomain]++;
             }
           }
         }
         return false;
       };
       auto progress_fn = [&](uint64_t, uint64_t, uint64_t current_value, uint64_t) -> void {
-        uint64_t num_found = num_valid_serial_numbers.load();
+        fprintf(stderr, "... %08" PRIX64 " %" PRId64 " mismatches; counts: [%zu/%zu, %zu/%zu, %zu/%zu, %zu/%zu, %zu/%zu, %zu/%zu, %zu/%zu, %zu/%zu, %zu/%zu]\r", current_value, num_mismatches.load(),
-        fprintf(stderr, "... %08" PRIX64 " %" PRId64 " (0x%" PRIX64 ") found\r",
+            found_counts[0].load(), serial_numbers[0].size(),
-            current_value, num_found, num_found);
+            found_counts[1].load(), serial_numbers[1].size(),
+            found_counts[2].load(), serial_numbers[2].size(),
+            found_counts[3].load(), serial_numbers[3].size(),
+            found_counts[4].load(), serial_numbers[4].size(),
+            found_counts[5].load(), serial_numbers[5].size(),
+            found_counts[6].load(), serial_numbers[6].size(),
+            found_counts[7].load(), serial_numbers[7].size(),
+            found_counts[8].load(), serial_numbers[8].size());
       };
-      phosg::parallel_range<uint64_t>(thread_fn, 0, 0x100000000, num_threads, progress_fn);
+      phosg::parallel_range_blocks<uint64_t>(thread_fn, 0, 0x100000000, 0x1000, num_threads, progress_fn);
+
+      if (num_mismatches > 0) {
+        throw logic_error("mismatches occurred during test");
+      }
     });
+
 Action a_inspect_dc_serial_number(
     "inspect-dc-serial-number", "\
   inspect-dc-serial-number SERIAL-NUMBER\n\
@@ -2797,7 +2979,7 @@ Action a_replay_ep3_battle_commands(
         run_replay(base_seed, 0);
       } else {
         size_t num_threads = args.get<size_t>("threads", 0);
-        phosg::parallel_range<int64_t>(run_replay, 0, 0x100000000, num_threads);
+        phosg::parallel_range_blocks<int64_t>(run_replay, 0, 0x100000000, 0x1000, num_threads);
       }
     });
 

src/Quest.cc

@@ -166,7 +166,7 @@ string find_seed_and_decrypt_download_quest_data_section(
     const void* data_section, size_t size, bool skip_checksum, bool is_ep3_trial, size_t num_threads) {
   mutex result_lock;
   string result;
-  uint64_t result_seed = phosg::parallel_range<uint64_t>([&](uint64_t seed, size_t) {
+  uint64_t result_seed = phosg::parallel_range_blocks<uint64_t>([&](uint64_t seed, size_t) {
     try {
       string ret = decrypt_download_quest_data_section<BE>(
           data_section, size, seed, skip_checksum, is_ep3_trial);
@@ -177,7 +177,7 @@ string find_seed_and_decrypt_download_quest_data_section(
       return false;
     }
   },
-      0, 0x100000000, num_threads);
+      0, 0x100000000, 0x1000, num_threads);
 
   if (!result.empty() && (result_seed < 0x100000000)) {
     static_game_data_log.info("Found seed %08" PRIX64, result_seed);

src/ReceiveCommands.cc

@@ -3416,7 +3416,7 @@ static void on_30(shared_ptr<Client> c, uint16_t, uint32_t, string& data) {
   shared_ptr<PSOBBCharacterFile> bb_char;
   switch (c->version()) {
     case Version::DC_V2:
-      bb_char = PSOBBCharacterFile::create_from_dc_v2(check_size_t<PSODCV2CharacterFile>(data));
+      bb_char = PSOBBCharacterFile::create_from_dc_v2(check_size_t<PSODCV2CharacterFile::Character>(data));
       break;
     case Version::GC_NTE:
       bb_char = PSOBBCharacterFile::create_from_gc_nte(check_size_t<PSOGCNTECharacterFileCharacter>(data));

src/SaveFileFormats.cc

@@ -143,6 +143,16 @@ bool PSOVMSFileHeader::checksum_correct() const {
   return (crc == this->crc);
 }
 
+void PSOVMSFileHeader::check() const {
+  if (!this->checksum_correct()) {
+    throw runtime_error("VMS file unencrypted header checksum is incorrect");
+  }
+}
+
+bool PSOVMSFileHeader::is_v2() const {
+  return !memcmp(this->short_desc.data, "PSOV2", 5);
+}
+
 bool PSOGCIFileHeader::checksum_correct() const {
   uint32_t cs = phosg::crc32(&this->game_name, this->game_name.bytes());
   cs = phosg::crc32(&this->embedded_seed, sizeof(this->embedded_seed), cs);
@@ -543,7 +553,7 @@ shared_ptr<PSOBBCharacterFile> PSOBBCharacterFile::create_from_preview(
       guild_card_number, language, preview.visual, preview.name.decode(language), level_table);
 }
 
-shared_ptr<PSOBBCharacterFile> PSOBBCharacterFile::create_from_dc_v2(const PSODCV2CharacterFile& dc) {
+shared_ptr<PSOBBCharacterFile> PSOBBCharacterFile::create_from_dc_v2(const PSODCV2CharacterFile::Character& dc) {
   auto ret = make_shared<PSOBBCharacterFile>();
   ret->inventory = dc.inventory;
   ret->inventory.decode_from_client(Version::DC_V2);
@@ -792,10 +802,10 @@ void save_psochar(
   phosg::fwritex(f.get(), empty_membership);
 }
 
-PSODCV2CharacterFile PSOBBCharacterFile::to_dc_v2() const {
+PSODCV2CharacterFile::Character PSOBBCharacterFile::to_dc_v2() const {
   uint8_t language = this->inventory.language;
 
-  PSODCV2CharacterFile ret;
+  PSODCV2CharacterFile::Character ret;
   ret.inventory = this->inventory;
   // We don't need to do the v1-compatible encoding (hence it is OK to pass
   // nullptr here) but we do need to encode mag stats in the v2 format

src/SaveFileFormats.hh

@@ -36,6 +36,12 @@ struct PSOVMSFileHeader {
   /* 0080 */ // parray<parray<uint8_t, 0x200>, num_icons> icon;
 
   bool checksum_correct() const;
+  void check() const;
+  inline size_t icon_data_size() const {
+    return this->num_icons * 0x200;
+  }
+
+  bool is_v2() const;
 } __packed_ws__(PSOVMSFileHeader, 0x80);
 
 struct PSOGCIFileHeader {
@@ -292,6 +298,8 @@ struct PSOBBBaseSystemFile : PSOBBMinimalSystemFile {
 // Character files
 
 struct PSODCNTECharacterFile {
+  /* 0000 */ le_uint32_t checksum = 0;
+  struct Character {
     // See PSOGCCharacterFile::Character for descriptions of fields' meanings.
     /* 0000:---- */ PlayerInventory inventory;
     /* 034C:---- */ PlayerDispDataDCPCV3 disp;
@@ -316,9 +324,15 @@ struct PSODCNTECharacterFile {
     /* 0A90:0674 */ pstring<TextEncoding::ASCII, 0x10> v1_serial_number;
     /* 0AA0:0684 */ pstring<TextEncoding::ASCII, 0x10> v1_access_key;
     /* 0AB0:0694 */
-} __packed_ws__(PSODCNTECharacterFile, 0xAB0);
+  } __packed_ws__(Character, 0xAB0);
+  /* 0004 */ Character character;
+  /* 0AB4 */ le_uint32_t round2_seed = 0;
+  /* 0AB8 */
+} __packed_ws__(PSODCNTECharacterFile, 0xAB8);
 
 struct PSODC112000CharacterFile {
+  /* 0000 */ le_uint32_t checksum = 0;
+  struct Character {
     // See PSOGCCharacterFile::Character for descriptions of fields' meanings.
     /* 0000:---- */ PlayerInventory inventory;
     /* 034C:---- */ PlayerDispDataDCPCV3 disp;
@@ -345,9 +359,14 @@ struct PSODC112000CharacterFile {
     /* 13B4:0F98 */ pstring<TextEncoding::ASCII, 0x10> v1_serial_number;
     /* 13C4:0FA8 */ pstring<TextEncoding::ASCII, 0x10> v1_access_key;
     /* 13D4:0FB8 */
-} __packed_ws__(PSODC112000CharacterFile, 0x13D4);
+  } __packed_ws__(Character, 0x13D4);
+  /* 0004 */ Character character;
+  /* 13D8 */ le_uint32_t round2_seed = 0;
+} __packed_ws__(PSODC112000CharacterFile, 0x13DC);
 
 struct PSODCV1CharacterFile {
+  /* 0000 */ le_uint32_t checksum = 0;
+  struct Character {
     // See PSOGCCharacterFile::Character for descriptions of fields' meanings.
     /* 0000:---- */ PlayerInventory inventory;
     /* 034C:---- */ PlayerDispDataDCPCV3 disp;
@@ -368,9 +387,14 @@ struct PSODCV1CharacterFile {
     /* 1528:110C */ pstring<TextEncoding::ASCII, 0x10> v1_serial_number;
     /* 1538:111C */ pstring<TextEncoding::ASCII, 0x10> v1_access_key;
     /* 1548:112C */
-} __packed_ws__(PSODCV1CharacterFile, 0x1548);
+  } __packed_ws__(Character, 0x1548);
+  /* 0004 */ Character character;
+  /* 154C */ le_uint32_t round2_seed = 0;
+} __packed_ws__(PSODCV1CharacterFile, 0x1550);
 
 struct PSODCV2CharacterFile {
+  /* 0000 */ le_uint32_t checksum = 0;
+  struct Character {
     // See PSOGCCharacterFile::Character for descriptions of fields' meanings.
     /* 0000:---- */ PlayerInventory inventory;
     /* 034C:---- */ PlayerDispDataDCPCV3 disp;
@@ -403,7 +427,10 @@ struct PSODCV2CharacterFile {
     /* 16D4:12B8 */ pstring<TextEncoding::ASCII, 0x10> v2_serial_number;
     /* 16E4:12C8 */ pstring<TextEncoding::ASCII, 0x10> v2_access_key;
     /* 16F4:12D8 */
-} __packed_ws__(PSODCV2CharacterFile, 0x16F4);
+  } __packed_ws__(Character, 0x16F4);
+  /* 0004 */ Character character;
+  /* 16F0 */ le_uint32_t round2_seed = 0;
+} __packed_ws__(PSODCV2CharacterFile, 0x16FC);
 
 struct PSOPCCharacterFile { // PSO______SYS and PSO______SYD
   // See PSOGCCharacterFile::Character for descriptions of fields' meanings.
@@ -517,7 +544,7 @@ struct PSOGCCharacterFile {
     //   ------zA BCDEFG-- HHHIIIJJ KLMNOPQR
     //   z = Function key setting (BB; 0 = menu shortcuts; 1 = chat shortcuts).
     //       This bit is unused by PSO GC.
-    //   A = Keyboard controls (BB; 0 = on; 1 = off). Note that A is also used
+    //   A = Keyboard controls (BB; 0 = on; 1 = off). This field is also used
     //       by PSO GC, but its function is currently unknown.
     //   G = Choice search setting (0 = enabled; 1 = disabled)
     //   H = Player lobby labels (0 = name; 1 = name, language, and level;
@@ -555,7 +582,7 @@ struct PSOGCCharacterFile {
     /* 2794:2378 */ be_uint32_t unknown_f8 = 0;
     /* 2798:237C */
   } __packed_ws__(Character, 0x2798);
-  /* 00004 */ parray<Character, 7> characters;
+  /* 00004 */ parray<Character, 7> characters; // 0-3: main chars, 4-6: temps
   /* 1152C */ pstring<TextEncoding::ASCII, 0x10> serial_number; // As %08X (not decimal)
   /* 1153C */ pstring<TextEncoding::ASCII, 0x10> access_key;
   /* 1154C */ pstring<TextEncoding::ASCII, 0x10> password;
@@ -760,13 +787,13 @@ struct PSOBBCharacterFile {
       uint8_t language,
       const PlayerDispDataBBPreview& preview,
       std::shared_ptr<const LevelTable> level_table);
-  static std::shared_ptr<PSOBBCharacterFile> create_from_dc_v2(const PSODCV2CharacterFile& dc);
+  static std::shared_ptr<PSOBBCharacterFile> create_from_dc_v2(const PSODCV2CharacterFile::Character& dc);
   static std::shared_ptr<PSOBBCharacterFile> create_from_gc_nte(const PSOGCNTECharacterFileCharacter& gc_nte);
   static std::shared_ptr<PSOBBCharacterFile> create_from_gc(const PSOGCCharacterFile::Character& gc);
   static std::shared_ptr<PSOBBCharacterFile> create_from_ep3(const PSOGCEp3CharacterFile::Character& ep3);
   static std::shared_ptr<PSOBBCharacterFile> create_from_xb(const PSOXBCharacterFileCharacter& xb);
 
-  PSODCV2CharacterFile to_dc_v2() const;
+  PSODCV2CharacterFile::Character to_dc_v2() const;
   PSOGCNTECharacterFileCharacter to_gc_nte() const;
   PSOGCCharacterFile::Character to_gc() const;
   PSOXBCharacterFileCharacter to_xb(uint64_t xb_user_id) const;
@@ -811,11 +838,45 @@ void save_psochar(
 ////////////////////////////////////////////////////////////////////////////////
 // Guild Card files
 
-struct PSODCV2GuildCardFile {
+struct PSODCNTEGuildCardFile {
-  // TODO: Fill this out.
+  // Note: DC NTE does not encrypt the Guild Card file
-  /* 0000 */ parray<uint8_t, 0x330C> unknown_a1;
+  struct Entry {
-  /* 330C */
+    /* 0000 */ GuildCardDCNTE guild_card;
-} __packed_ws__(PSODCV2GuildCardFile, 0x330C);
+    /* 007B */ uint8_t unknown_a1 = 0; // Probably actually unused
+    /* 007C */
+  } __packed_ws__(Entry, 0x7C);
+  /* 0000 */ parray<Entry, 100> entries;
+  /* 3070 */
+} __packed_ws__(PSODCNTEGuildCardFile, 0x3070);
+
+struct PSODCGuildCardFileEntry {
+  /* 0000 */ GuildCardDC guild_card;
+  /* 007D */ parray<uint8_t, 3> unknown_a1 = 0; // Probably actually unused
+  /* 0080 */
+} __packed_ws__(PSODCGuildCardFileEntry, 0x80);
+
+struct PSODC112000GuildCardFile {
+  // Note: 11/2000 does not encrypt the Guild Card file
+  /* 0000 */ parray<PSODCGuildCardFileEntry, 100> entries;
+  /* 3200 */
+} __packed_ws__(PSODC112000GuildCardFile, 0x3200);
+
+struct PSODCV1V2GuildCardFile {
+  struct EncryptedSection {
+    /* 0000 */ le_uint32_t checksum = 0;
+    /* 0004 */ parray<PSODCGuildCardFileEntry, 100> entries;
+    /* 3204 */ le_int16_t music_volume = 0;
+    /* 3206 */ int8_t sound_volume = 0;
+    /* 3207 */ uint8_t language = 1;
+    /* 3208 */ le_uint32_t server_time_delta_frames = 540000; // 648000 on DCv1
+    /* 320C */ le_uint32_t creation_timestamp = 0;
+    /* 3210 */ le_uint32_t round2_seed = 0;
+    /* 3214 */
+  } __packed_ws__(EncryptedSection, 0x3214);
+  /* 0000 */ EncryptedSection encrypted_section;
+  /* 3214 */ parray<uint8_t, 0x100> event_flags;
+  /* 3314 */
+} __packed_ws__(PSODCV1V2GuildCardFile, 0x3314);
 
 struct PSOPCGuildCardFile { // PSO______GUD
   /* 0000 */ le_uint32_t checksum = 0;
@@ -979,14 +1040,14 @@ std::string decrypt_fixed_size_data_section_s(
     size_t size,
     uint32_t round1_seed,
     bool skip_checksum = false,
-    uint64_t override_round2_seed = 0xFFFFFFFFFFFFFFFF) {
+    int64_t override_round2_seed = -1) {
   if (size < 2 * sizeof(U32T<BE>)) {
     throw std::runtime_error("data size is too small");
   }
   std::string decrypted = decrypt_data_section<BE>(data_section, size, round1_seed);
 
-  uint32_t round2_seed = override_round2_seed < 0x100000000
+  uint32_t round2_seed = (override_round2_seed >= 0)
-      ? static_cast<uint32_t>(override_round2_seed)
+      ? override_round2_seed
       : reinterpret_cast<const U32T<BE>*>(decrypted.data() + decrypted.size() - sizeof(U32T<BE>))->load();
   PSOV2Encryption round2_crypt(round2_seed);
   if (BE) {
@@ -1011,13 +1072,13 @@ std::string decrypt_fixed_size_data_section_s(
   return decrypted;
 }
 
-template <typename StructT, bool BE>
+template <typename StructT, bool BE, size_t ChecksumLength = sizeof(StructT)>
 StructT decrypt_fixed_size_data_section_t(
     const void* data_section,
     size_t size,
     uint32_t round1_seed,
     bool skip_checksum = false,
-    uint64_t override_round2_seed = 0xFFFFFFFFFFFFFFFF) {
+    int64_t override_round2_seed = -1) {
 
   std::string decrypted = decrypt_data_section<BE>(data_section, size, round1_seed);
   if (decrypted.size() < sizeof(StructT)) {
@@ -1025,7 +1086,7 @@ StructT decrypt_fixed_size_data_section_t(
   }
   StructT ret = *reinterpret_cast<const StructT*>(decrypted.data());
 
-  PSOV2Encryption round2_crypt(override_round2_seed < 0x100000000 ? override_round2_seed : ret.round2_seed.load());
+  PSOV2Encryption round2_crypt((override_round2_seed >= 0) ? override_round2_seed : ret.round2_seed.load());
   if (BE) {
     round2_crypt.encrypt_big_endian(&ret, offsetof(StructT, round2_seed));
   } else {
@@ -1035,7 +1096,7 @@ StructT decrypt_fixed_size_data_section_t(
   if (!skip_checksum) {
     uint32_t expected_crc = ret.checksum;
     ret.checksum = 0;
-    uint32_t actual_crc = phosg::crc32(&ret, sizeof(ret));
+    uint32_t actual_crc = phosg::crc32(&ret, ChecksumLength);
     ret.checksum = expected_crc;
     if (expected_crc != actual_crc) {
       throw std::runtime_error(phosg::string_printf(
@@ -1070,12 +1131,12 @@ std::string encrypt_fixed_size_data_section_s(const void* data, size_t size, uin
   return encrypt_data_section<BE>(encrypted.data(), encrypted.size(), round1_seed);
 }
 
-template <typename StructT, bool BE>
+template <typename StructT, bool BE, size_t ChecksumLength = sizeof(StructT)>
 std::string encrypt_fixed_size_data_section_t(const StructT& s, uint32_t round1_seed) {
   StructT encrypted = s;
   encrypted.checksum = 0;
   encrypted.round2_seed = phosg::random_object<uint32_t>();
-  encrypted.checksum = phosg::crc32(&encrypted, sizeof(encrypted));
+  encrypted.checksum = phosg::crc32(&encrypted, ChecksumLength);
 
   PSOV2Encryption round2_crypt(encrypted.round2_seed);
   if (BE) {

system/client-functions/ExtendedPlayerInfo/SetExtendedPlayerInfo.2OEF.patch.s

@@ -10,4 +10,4 @@ start:
 data:
   .data  0x8C4EC4E0  # char_file_part1
   .data  0x8C4EC4E4  # char_file_part2
-  # Server adds a PSODCV2CharacterFile here
+  # Server adds a PSODCV2CharacterFile::Character here

system/client-functions/ExtendedPlayerInfo/SetExtendedPlayerInfo.2OJ5.patch.s

@@ -10,4 +10,4 @@ start:
 data:
   .data  0x8C4EC4E0  # char_file_part1
   .data  0x8C4EC4E4  # char_file_part2
-  # Server adds a PSODCV2CharacterFile here
+  # Server adds a PSODCV2CharacterFile::Character here

system/client-functions/ExtendedPlayerInfo/SetExtendedPlayerInfo.2OJF.patch.s

@@ -10,4 +10,4 @@ start:
 data:
   .data  0x8C4E5F80  # char_file_part1
   .data  0x8C4E5F84  # char_file_part2
-  # Server adds a PSODCV2CharacterFile here
+  # Server adds a PSODCV2CharacterFile::Character here

system/client-functions/ExtendedPlayerInfo/SetExtendedPlayerInfo.2OPF.patch.s

@@ -10,4 +10,4 @@ start:
 data:
   .data  0x8C4DB9E0  # char_file_part1
   .data  0x8C4DB9E4  # char_file_part2
-  # Server adds a PSODCV2CharacterFile here
+  # Server adds a PSODCV2CharacterFile::Character here

tests/decrypt-vms-save.test.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -e
+
+EXECUTABLE="$1"
+if [ -z "$EXECUTABLE" ]; then
+  EXECUTABLE="./newserv"
+fi
+
+DIR="tests/saves-vms"
+
+echo "... decrypt v1 charfile"
+$EXECUTABLE decrypt-vms-save $DIR/d1-sys.vms $DIR/d1-sys.vmsd --serial-number=77777777
+diff $DIR/d1-sys-expected.vmsd $DIR/d1-sys.vmsd
+
+echo "... decrypt v2 charfile"
+$EXECUTABLE decrypt-vms-save $DIR/e1-sys.vms $DIR/e1-sys.vmsd --serial-number=DBA61FAC
+diff $DIR/e1-sys-expected.vmsd $DIR/e1-sys.vmsd
+
+echo "... decrypt v1/v2 guildfile"
+$EXECUTABLE decrypt-vms-save $DIR/a1-2gc.vms $DIR/a1-2gc.vmsd --serial-number=DBA61FAC
+diff $DIR/a1-2gc-expected.vmsd $DIR/a1-2gc.vmsd
+
+echo "... clean up"
+rm -f $DIR/d1-sys.vmsd $DIR/e1-sys.vmsd $DIR/a1-2gc.vmsd