support extended attributes in json rare tables

src/ItemCreator.cc

@@ -272,12 +272,16 @@ ItemData ItemCreator::check_rare_specs_and_create_rare_box_item(uint8_t area_nor
   for (const auto& spec : rare_specs) {
     item = this->check_rate_and_create_rare_item(spec, area_norm);
     if (!item.empty()) {
-      this->log.info("Box spec %08" PRIX32 " produced item %02hhX%02hhX%02hhX",
+      if (this->log.should_log(LogLevel::INFO)) {
-          spec.probability, spec.item_code[0], spec.item_code[1], spec.item_code[2]);
+        auto hex = spec.data.hex();
+        this->log.info("Box spec %08" PRIX32 " produced item %s", spec.probability, hex.c_str());
+      }
       break;
     }
-    this->log.info("Box spec %08" PRIX32 " did not produce item %02hhX%02hhX%02hhX",
+    if (this->log.should_log(LogLevel::INFO)) {
-        spec.probability, spec.item_code[0], spec.item_code[1], spec.item_code[2]);
+      auto hex = spec.data.hex();
+      this->log.info("Box spec %08" PRIX32 " did not produce item %s", spec.probability, hex.c_str());
+    }
   }
   return item;
 }
@@ -329,12 +333,16 @@ ItemData ItemCreator::check_rare_spec_and_create_rare_enemy_item(uint32_t enemy_
     for (const auto& spec : rare_specs) {
       item = this->check_rate_and_create_rare_item(spec, area_norm);
       if (!item.empty()) {
-        this->log.info("Enemy spec %08" PRIX32 " produced item %02hhX%02hhX%02hhX",
+        if (this->log.should_log(LogLevel::INFO)) {
-            spec.probability, spec.item_code[0], spec.item_code[1], spec.item_code[2]);
+          auto hex = spec.data.hex();
+          this->log.info("Enemy spec %08" PRIX32 " produced item %s", spec.probability, hex.c_str());
+        }
         break;
       }
-      this->log.info("Enemy spec %08" PRIX32 " did not produce item %02hhX%02hhX%02hhX",
+      if (this->log.should_log(LogLevel::INFO)) {
-          spec.probability, spec.item_code[0], spec.item_code[1], spec.item_code[2]);
+        auto hex = spec.data.hex();
+        this->log.info("Enemy spec %08" PRIX32 " did not produce item %s", spec.probability, hex.c_str());
+      }
     }
   }
   return item;
@@ -351,37 +359,36 @@ ItemData ItemCreator::check_rate_and_create_rare_item(const RareItemSet::Expande
     return ItemData();
   }
 
-  ItemData item;
+  ItemData item = drop.data;
-  item.data1[0] = drop.item_code[0];
+  if (item.can_be_encoded_in_rel_rare_table()) {
-  item.data1[1] = drop.item_code[1];
+    switch (item.data1[0]) {
-  item.data1[2] = drop.item_code[2];
+      case 0:
-  switch (item.data1[0]) {
+        if (this->pt->has_rare_bonus_value_prob_table) {
-    case 0:
+          this->generate_rare_weapon_bonuses(item, this->rand_int(10));
-      if (this->pt->has_rare_bonus_value_prob_table) {
+        } else {
-        this->generate_rare_weapon_bonuses(item, this->rand_int(10));
+          this->generate_common_weapon_bonuses(item, area_norm);
-      } else {
+        }
-        this->generate_common_weapon_bonuses(item, area_norm);
+        this->set_item_unidentified_flag_if_not_challenge(item);
-      }
+        break;
-      this->set_item_unidentified_flag_if_not_challenge(item);
+      case 1:
-      break;
+        this->generate_common_armor_slots_and_bonuses(item);
-    case 1:
+        break;
-      this->generate_common_armor_slots_and_bonuses(item);
+      case 2:
-      break;
+        this->generate_common_mag_variances(item);
-    case 2:
+        break;
-      this->generate_common_mag_variances(item);
+      case 3:
-      break;
+        this->clear_tool_item_if_invalid(item);
-    case 3:
+        this->set_tool_item_amount_to_1(item);
-      this->clear_tool_item_if_invalid(item);
+        break;
-      this->set_tool_item_amount_to_1(item);
+      case 4:
-      break;
+        break;
-    case 4:
+      default:
-      break;
+        throw logic_error("invalid item class");
-    default:
+    }
-      throw logic_error("invalid item class");
+    this->set_item_kill_count_if_unsealable(item);
   }
 
   this->clear_item_if_restricted(item);
-  this->set_item_kill_count_if_unsealable(item);
   return item;
 }
 

src/ItemData.cc

@@ -665,6 +665,10 @@ bool ItemData::can_be_equipped_in_slot(EquipSlot slot) const {
   }
 }
 
+bool ItemData::can_be_encoded_in_rel_rare_table() const {
+  return !(this->data1[3] || this->data1d[1] || this->data1d[2] || this->data2d);
+}
+
 bool ItemData::compare_for_sort(const ItemData& a, const ItemData& b) {
   for (size_t z = 0; z < 12; z++) {
     if (a.data1[z] < b.data1[z]) {
@@ -722,10 +726,19 @@ ItemData ItemData::from_primary_identifier(const StackLimits& limits, uint32_t p
 }
 
 string ItemData::hex() const {
-  return string_printf("%02hhX%02hhX%02hhX%02hhX %02hhX%02hhX%02hhX%02hhX %02hhX%02hhX%02hhX%02hhX (%08" PRIX32 ") %02hhX%02hhX%02hhX%02hhX",
+  return string_printf("%08" PRIX32 " %08" PRIX32 " %08" PRIX32 " (%08" PRIX32 ") %08" PRIX32,
-      this->data1[0], this->data1[1], this->data1[2], this->data1[3],
+      this->data1db[0].load(), this->data1db[1].load(), this->data1db[2].load(), this->id.load(), this->data2db.load());
-      this->data1[4], this->data1[5], this->data1[6], this->data1[7],
+}
-      this->data1[8], this->data1[9], this->data1[10], this->data1[11],
+
-      this->id.load(),
+string ItemData::short_hex() const {
-      this->data2[0], this->data2[1], this->data2[2], this->data2[3]);
+  auto ret = string_printf("%08" PRIX32 "%08" PRIX32 "%08" PRIX32 "%08" PRIX32,
+      this->data1db[0].load(), this->data1db[1].load(), this->data1db[2].load(), this->data2db.load());
+  size_t offset = ret.find_last_not_of('0');
+  if (offset != string::npos) {
+    offset += (offset & 1) ? 1 : 2;
+    if (offset < ret.size()) {
+      ret.resize(offset);
+    }
+  }
+  return ret;
 }

src/ItemData.hh

@@ -146,6 +146,7 @@ struct ItemData {
   static ItemData from_data(const std::string& data);
   static ItemData from_primary_identifier(const StackLimits& limits, uint32_t primary_identifier);
   std::string hex() const;
+  std::string short_hex() const;
   uint32_t primary_identifier() const;
 
   bool is_wrapped(const StackLimits& limits) const;
@@ -189,6 +190,8 @@ struct ItemData {
   EquipSlot default_equip_slot() const;
   bool can_be_equipped_in_slot(EquipSlot slot) const;
 
+  bool can_be_encoded_in_rel_rare_table() const;
+
   bool empty() const;
 
   static bool compare_for_sort(const ItemData& a, const ItemData& b);

src/ItemNameIndex.cc

@@ -408,6 +408,16 @@ ItemData ItemNameIndex::parse_item_description_phase(const std::string& descript
   if (is_wrapped) {
     desc = desc.substr(8);
   }
+  bool is_unidentified = starts_with(desc, "?");
+  if (is_unidentified) {
+    size_t z;
+    for (z = 1; z < desc.size(); z++) {
+      if (desc[z] != ' ' && desc[z] != '?') {
+        break;
+      }
+    }
+    desc = desc.substr(z);
+  }
 
   // TODO: It'd be nice to be able to parse S-rank weapon specials here too.
   uint8_t weapon_special = 0;
@@ -459,7 +469,7 @@ ItemData ItemNameIndex::parse_item_description_phase(const std::string& descript
 
   if (ret.data1[0] == 0x00) {
     // Weapons: add special, grind and percentages (or name, if S-rank)
-    ret.data1[4] = weapon_special | (is_wrapped ? 0x40 : 0x00);
+    ret.data1[4] = weapon_special | (is_wrapped ? 0x40 : 0x00) | (is_unidentified ? 0x80 : 0x00);
 
     auto tokens = split(desc, ' ');
     for (auto& token : tokens) {

src/Main.cc

@@ -1618,7 +1618,8 @@ Action a_convert_rare_item_set(
     +[](Arguments& args) {
       auto version = get_cli_version(args);
 
-      auto s = make_shared<ServerState>();
+      auto s = make_shared<ServerState>("system/config.json");
+      s->load_config_early();
       s->load_patch_indexes(false);
       s->load_text_index(false);
       s->load_item_definitions(false);

src/RareItemSet.cc

@@ -12,20 +12,16 @@ using namespace std;
 
 string RareItemSet::ExpandedDrop::str() const {
   auto frac = reduce_fraction<uint64_t>(this->probability, 0x100000000);
+  auto hex = this->data.hex();
   return string_printf(
-      "(%08" PRIX32 " => %" PRIu64 "/%" PRIu64 ") %02hhX%02hhX%02hhX",
+      "(%08" PRIX32 " => %" PRIu64 "/%" PRIu64 ") %s",
-      this->probability, frac.first, frac.second, this->item_code[0], this->item_code[1], this->item_code[2]);
+      this->probability, frac.first, frac.second, hex.c_str());
 }
 
 string RareItemSet::ExpandedDrop::str(shared_ptr<const ItemNameIndex> name_index) const {
-  ItemData item;
-  item.data1[0] = this->item_code[0];
-  item.data1[1] = this->item_code[1];
-  item.data1[2] = this->item_code[2];
-
   string ret = this->str();
   ret += " (";
-  ret += name_index->describe_item(item);
+  ret += name_index->describe_item(this->data);
   ret += ")";
   return ret;
 }
@@ -78,14 +74,22 @@ uint8_t RareItemSet::compress_rate(uint32_t probability) {
 }
 
 RareItemSet::ParsedRELData::PackedDrop::PackedDrop(const ExpandedDrop& exp)
-    : probability(RareItemSet::compress_rate(exp.probability)),
+    : probability(RareItemSet::compress_rate(exp.probability)) {
-      item_code(exp.item_code) {}
+  if (!exp.data.can_be_encoded_in_rel_rare_table()) {
+    throw runtime_error("item " + exp.data.short_hex() + " has extended attributes and cannot be encoded in a REL file");
+  }
+  this->item_code[0] = exp.data.data1[0];
+  this->item_code[1] = exp.data.data1[1];
+  this->item_code[2] = exp.data.data1[2];
+}
 
 RareItemSet::ExpandedDrop RareItemSet::ParsedRELData::PackedDrop::expand() const {
-  return ExpandedDrop{
+  ExpandedDrop ret;
-      .probability = RareItemSet::expand_rate(this->probability),
+  ret.probability = RareItemSet::expand_rate(this->probability);
-      .item_code = this->item_code,
+  ret.data.data1[0] = this->item_code[0];
-  };
+  ret.data.data1[1] = this->item_code[1];
+  ret.data.data1[2] = this->item_code[2];
+  return ret;
 }
 
 template <bool IsBigEndian>
@@ -184,10 +188,9 @@ RareItemSet::ParsedRELData::ParsedRELData(const SpecCollection& collection) {
   for (const auto& specs : collection.rt_index_to_specs) {
     ExpandedDrop effective_spec;
     for (const auto& spec : specs) {
-      if (effective_spec.item_code.is_filled_with(0)) {
+      if (effective_spec.data.empty()) {
         effective_spec = spec;
-      } else if ((effective_spec.probability != spec.probability) ||
+      } else if ((effective_spec.probability != spec.probability) || (effective_spec.data != spec.data)) {
-          (effective_spec.item_code != spec.item_code)) {
         throw runtime_error("monster spec cannot be converted to ItemRT format");
       }
     }
@@ -216,7 +219,7 @@ RareItemSet::SpecCollection RareItemSet::ParsedRELData::as_collection() const {
   SpecCollection ret;
   for (size_t z = 0; z < this->monster_rares.size(); z++) {
     const auto& drop = this->monster_rares[z];
-    if (drop.item_code.is_filled_with(0)) {
+    if (drop.data.empty()) {
       continue;
     }
     if (z >= ret.rt_index_to_specs.size()) {
@@ -225,7 +228,7 @@ RareItemSet::SpecCollection RareItemSet::ParsedRELData::as_collection() const {
     ret.rt_index_to_specs[z].emplace_back(drop);
   }
   for (const auto& drop : this->box_rares) {
-    if (drop.drop.item_code.is_filled_with(0)) {
+    if (drop.drop.data.empty()) {
       continue;
     }
     if (drop.area >= ret.box_area_to_specs.size()) {
@@ -362,17 +365,14 @@ RareItemSet::RareItemSet(const JSON& json, shared_ptr<const ItemNameIndex> name_
               auto item_desc = spec_json->at(1);
               if (item_desc.is_int()) {
                 uint32_t item_code = item_desc.as_int();
-                d.item_code[0] = (item_code >> 16) & 0xFF;
+                d.data.data1[0] = (item_code >> 16) & 0xFF;
-                d.item_code[1] = (item_code >> 8) & 0xFF;
+                d.data.data1[1] = (item_code >> 8) & 0xFF;
-                d.item_code[2] = item_code & 0xFF;
+                d.data.data1[2] = item_code & 0xFF;
               } else if (item_desc.is_string()) {
                 if (!name_index) {
                   throw runtime_error("item name index is not available");
                 }
-                ItemData data = name_index->parse_item_description(item_desc.as_string());
+                d.data = name_index->parse_item_description(item_desc.as_string());
-                d.item_code[0] = data.data1[0];
-                d.item_code[1] = data.data1[1];
-                d.item_code[2] = data.data1[2];
               } else {
                 throw runtime_error("invalid item description type");
               }
@@ -446,19 +446,18 @@ JSON RareItemSet::json(shared_ptr<const ItemNameIndex> name_index) const {
             }
 
             for (const auto& spec : this->get_enemy_specs(GameMode::NORMAL, episode, difficulty, section_id, rt_index)) {
-              uint32_t data1_0_1_2 = (spec.item_code[0] << 16) | (spec.item_code[1] << 8) | spec.item_code[2];
+              if (spec.data.empty()) {
-              if (data1_0_1_2 == 0) {
                 continue;
               }
-
               auto frac = reduce_fraction<uint64_t>(spec.probability, 0x100000000);
-              auto spec_json = JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second), data1_0_1_2});
+              auto spec_json = JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second)});
+              if (spec.data.can_be_encoded_in_rel_rare_table()) {
+                spec_json.emplace_back((spec.data.data1[0] << 16) | (spec.data.data1[1] << 8) | spec.data.data1[2]);
+              } else {
+                spec_json.emplace_back(spec.data.short_hex());
+              }
               if (name_index) {
-                ItemData data;
+                spec_json.emplace_back(name_index->describe_item(spec.data));
-                data.data1[0] = spec.item_code[0];
-                data.data1[1] = spec.item_code[1];
-                data.data1[2] = spec.item_code[2];
-                spec_json.emplace_back(name_index->describe_item(data));
               }
               for (const auto& enemy_type : enemy_types) {
                 if (enemy_type_valid_for_episode(episode, enemy_type)) {
@@ -473,20 +472,20 @@ JSON RareItemSet::json(shared_ptr<const ItemNameIndex> name_index) const {
             auto area_list = JSON::list();
 
             for (const auto& spec : this->get_box_specs(GameMode::NORMAL, episode, difficulty, section_id, area)) {
-              uint32_t data1_0_1_2 = (spec.item_code[0] << 16) | (spec.item_code[1] << 8) | spec.item_code[2];
+              if (spec.data.empty()) {
-              if (data1_0_1_2 == 0) {
                 continue;
               }
-
               auto frac = reduce_fraction<uint64_t>(spec.probability, 0x100000000);
-              area_list.emplace_back(JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second), data1_0_1_2}));
+              auto spec_json = JSON::list({string_printf("%" PRIu64 "/%" PRIu64, frac.first, frac.second)});
-              if (name_index) {
+              if (spec.data.can_be_encoded_in_rel_rare_table()) {
-                ItemData data;
+                spec_json.emplace_back((spec.data.data1[0] << 16) | (spec.data.data1[1] << 8) | spec.data.data1[2]);
-                data.data1[0] = spec.item_code[0];
+              } else {
-                data.data1[1] = spec.item_code[1];
+                spec_json.emplace_back(spec.data.short_hex());
-                data.data1[2] = spec.item_code[2];
-                area_list.back().emplace_back(name_index->describe_item(data));
               }
+              if (name_index) {
+                spec_json.emplace_back(name_index->describe_item(spec.data));
+              }
+              area_list.emplace_back(std::move(spec_json));
             }
 
             if (!area_list.empty()) {

src/RareItemSet.hh

@@ -19,7 +19,7 @@ class RareItemSet {
 public:
   struct ExpandedDrop {
     uint32_t probability = 0;
-    parray<uint8_t, 3> item_code;
+    ItemData data;
 
     std::string str() const;
     std::string str(std::shared_ptr<const ItemNameIndex> name_index) const;

system/item-tables/rare-table-v4.json

@@ -10,11 +10,15 @@
   // Probability may be a 32-bit integer specifying the relative frequency of
   // finding the item, out of 2^32 (so 0x80000000 means a 50% chance), or it may
   // be a fraction represented as a string (e.g. "3/32"). ItemDesc may be a
-  // textual description of the item, or it may be an integer specifying the
+  // textual description of the item, the item's data specified as a hex string,
-  // 3-byte item code (in this case, the item code may be specified in hex, like
+  // or an integer specifying the 3-byte item code (in this case, the item code
-  // 0x009D00). Keep in mind that only the first 3 bytes of the item code are
+  // may be specified in hex, like 0x009D00). If an item has any extended
-  // used, so weapon grinds and attributes, etc. will be ignored when specifying
+  // attributes specified (that is, if there are any nonzero bytes in the item's
-  // items as strings.
+  // data beyond the first three bytes), then the standard random attribute
+  // logic is disabled for that item and it will drop exactly as specified.
+  // Furthermore, if any item has any extended attributes specified, then the
+  // entire rare table can only be represented in this JSON format and cannot be
+  // converted to any other format (AFS/GSL/REL).
   "Normal": {
     "Episode1": {
       "Hard": {