incentive/psopeeps-newserv
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add MapFile::serialize

Browse Source
This commit is contained in:
Martin Michelsen
2025-12-20 18:52:45 -08:00
parent 0a4c9a0a61
commit a9fa138213
21 changed files with 309 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Main.cc
+97 -68
View File
@@ -3074,8 +3074,8 @@ Action a_materialize_map(
Runs the Challenge Mode random enemy generation algorithm on the input map\n\
file, producing a new map file with no random sections. A version option\n\
is required, and the --seed=SEED option is also required (SEED is a 32-bit\n\
hex integer). The resulting map file is disassembled immediately, and the\n\
disassembly is written to the output file.\n",
hex integer). If --disassemble is given, disassembles the result instead\n\
of generating the map data.\n",
+[](phosg::Arguments& args) {
if (args.get<bool>("debug")) {
static_game_data_log.min_level = phosg::LogLevel::L_DEBUG;
@@ -3083,10 +3083,15 @@ Action a_materialize_map(
auto map_data = make_shared<string>(prs_decompress(read_input_data(args)));
auto map_file = make_shared<MapFile>(map_data);
uint32_t seed = args.get<uint32_t>("seed", phosg::Arguments::IntFormat::HEX);
Version version = get_cli_version(args);
auto materialized = map_file->materialize_random_sections(seed);
auto disassembly = materialized->disassemble(false, version);
write_output_data(args, disassembly.data(), disassembly.size(), "txt");
if (args.get<bool>("disassemble")) {
Version version = get_cli_version(args);
auto disassembly = materialized->disassemble(false, version);
write_output_data(args, disassembly.data(), disassembly.size(), "txt");
} else {
auto new_data = prs_compress_optimal(materialized->serialize());
write_output_data(args, new_data.data(), new_data.size(), "dat");
}
});
Action a_print_free_supermap(
@@ -3174,7 +3179,7 @@ Action a_print_free_supermap(
map_state.print(stdout);
});
Action a_check_quest_reassembly(
Action a_check_quests(
"check-quests", nullptr,
+[](phosg::Arguments& args) {
check_quest_opcode_definitions();
@@ -3188,72 +3193,96 @@ Action a_check_quest_reassembly(
s->load_maps();
s->load_quest_index(true);
if (args.get<bool>("reassembly")) {
bool reassemble_scripts = args.get<bool>("reassemble-scripts");
bool reassemble_maps = args.get<bool>("reassemble-maps");
if (reassemble_scripts || reassemble_maps) {
for (const auto& [_, q] : s->quest_index->quests_by_number) {
for (const auto& [_, vq] : q->versions) {
auto decompressed_bin = prs_decompress(*vq->bin_contents);
auto disassembled = disassemble_quest_script(decompressed_bin.data(), decompressed_bin.size(), vq->meta.version, vq->meta.language, vq->map_file, false, false);
auto reassembly = disassemble_quest_script(decompressed_bin.data(), decompressed_bin.size(), vq->meta.version, vq->meta.language, vq->map_file, true, false);
string include_dir = phosg::dirname(vq->bin_filename());
AssembledQuestScript assembled;
try {
assembled = assemble_quest_script(
reassembly,
{"system/quests/includes"},
{"system/quests/includes", "system/client-functions/System"},
false);
if (vq->json_contents) {
assembled.meta.apply_json_overrides(*vq->json_contents);
if (reassemble_maps) {
auto dat = prs_decompress(*vq->dat_contents);
auto serialized = vq->map_file->serialize();
if (dat != serialized) {
phosg::log_info_f("... DISASSEMBLY:");
phosg::fwritex(stdout, vq->map_file->disassemble(false, vq->meta.version));
phosg::log_info_f("... BINDIFF:");
phosg::print_binary_diff(
stdout, dat.data(), dat.size(), serialized.data(), serialized.size(), isatty(fileno(stdout)), 3);
phosg::log_info_f("... {} {} {} ({}) MAP FAILED",
phosg::name_for_enum(vq->meta.version),
name_for_language(vq->meta.language),
vq->dat_filename(),
vq->meta.name);
throw std::runtime_error("re-serialized map file differs from original");
}
if (assembled.data != decompressed_bin) {
throw std::runtime_error("Reassembled quest script does not match original");
}
// Don't check quest number, since we override it based on the filename
if (assembled.meta.version != vq->meta.version) {
throw std::runtime_error(std::format("Reassembled quest version ({}) does not match original ({})",
phosg::name_for_enum(assembled.meta.version), phosg::name_for_enum(vq->meta.version)));
}
if (assembled.meta.language != vq->meta.language) {
throw std::runtime_error(std::format("Reassembled quest language ({}) does not match original ({})",
name_for_language(assembled.meta.language), name_for_language(vq->meta.language)));
}
if (assembled.meta.episode != vq->meta.episode) {
throw std::runtime_error(std::format("Reassembled quest episode ({}) does not match original ({})",
name_for_episode(assembled.meta.episode), name_for_episode(vq->meta.episode)));
}
if (assembled.meta.joinable != vq->meta.joinable) {
throw std::runtime_error(std::format("Reassembled quest joinable ({}) does not match original ({})",
assembled.meta.joinable, vq->meta.joinable));
}
if (assembled.meta.max_players != vq->meta.max_players) {
throw std::runtime_error(std::format("Reassembled quest max_players ({}) does not match original ({})",
assembled.meta.max_players, vq->meta.max_players));
}
if (assembled.meta.name != vq->meta.name) {
throw std::runtime_error(std::format("Reassembled quest name ({}) does not match original ({})",
assembled.meta.name, vq->meta.name));
}
if (assembled.meta.short_description != vq->meta.short_description) {
throw std::runtime_error(std::format("Reassembled quest short description ({}) does not match original ({})",
assembled.meta.short_description, vq->meta.short_description));
}
if (assembled.meta.long_description != vq->meta.long_description) {
throw std::runtime_error(std::format("Reassembled quest long description ({}) does not match original ({})",
assembled.meta.long_description, vq->meta.long_description));
}
} catch (const std::exception& e) {
phosg::log_error_f("================ DISASSEMBLY:");
phosg::fwritex(stderr, disassembled);
phosg::log_error_f("================ REASSEMBLY:");
phosg::fwritex(stderr, reassembly);
if (!assembled.data.empty()) {
phosg::log_error_f("================ BINDIFF:");
phosg::print_binary_diff(stderr, decompressed_bin.data(), decompressed_bin.size(), assembled.data.data(), assembled.data.size(), isatty(fileno(stderr)), 3, 0);
}
phosg::log_info_f("... {} {} {} ({}) FAILED", phosg::name_for_enum(vq->meta.version), name_for_language(vq->meta.language), vq->bin_filename(), vq->meta.name);
throw;
phosg::log_info_f("... {} {} {} ({}) MAP OK", phosg::name_for_enum(vq->meta.version), name_for_language(vq->meta.language), vq->dat_filename(), vq->meta.name);
}
if (reassemble_scripts) {
auto bin = prs_decompress(*vq->bin_contents);
auto disassembled = disassemble_quest_script(
bin.data(), bin.size(), vq->meta.version, vq->meta.language, vq->map_file, false, false);
auto reassembly = disassemble_quest_script(
bin.data(), bin.size(), vq->meta.version, vq->meta.language, vq->map_file, true, false);
string include_dir = phosg::dirname(vq->bin_filename());
AssembledQuestScript assembled;
try {
assembled = assemble_quest_script(
reassembly,
{"system/quests/includes"},
{"system/quests/includes", "system/client-functions/System"},
false);
if (vq->json_contents) {
assembled.meta.apply_json_overrides(*vq->json_contents);
}
if (assembled.data != bin) {
throw std::runtime_error("Reassembled quest script does not match original");
}
// Don't check quest number, since we override it based on the filename
if (assembled.meta.version != vq->meta.version) {
throw std::runtime_error(std::format("Reassembled quest version ({}) does not match original ({})",
phosg::name_for_enum(assembled.meta.version), phosg::name_for_enum(vq->meta.version)));
}
if (assembled.meta.language != vq->meta.language) {
throw std::runtime_error(std::format("Reassembled quest language ({}) does not match original ({})",
name_for_language(assembled.meta.language), name_for_language(vq->meta.language)));
}
if (assembled.meta.episode != vq->meta.episode) {
throw std::runtime_error(std::format("Reassembled quest episode ({}) does not match original ({})",
name_for_episode(assembled.meta.episode), name_for_episode(vq->meta.episode)));
}
if (assembled.meta.joinable != vq->meta.joinable) {
throw std::runtime_error(std::format("Reassembled quest joinable ({}) does not match original ({})",
assembled.meta.joinable, vq->meta.joinable));
}
if (assembled.meta.max_players != vq->meta.max_players) {
throw std::runtime_error(std::format("Reassembled quest max_players ({}) does not match original ({})",
assembled.meta.max_players, vq->meta.max_players));
}
if (assembled.meta.name != vq->meta.name) {
throw std::runtime_error(std::format("Reassembled quest name ({}) does not match original ({})",
assembled.meta.name, vq->meta.name));
}
if (assembled.meta.short_description != vq->meta.short_description) {
throw std::runtime_error(std::format("Reassembled quest short description ({}) does not match original ({})",
assembled.meta.short_description, vq->meta.short_description));
}
if (assembled.meta.long_description != vq->meta.long_description) {
throw std::runtime_error(std::format("Reassembled quest long description ({}) does not match original ({})",
assembled.meta.long_description, vq->meta.long_description));
}
} catch (const std::exception& e) {
phosg::log_error_f("================ DISASSEMBLY:");
phosg::fwritex(stderr, disassembled);
phosg::log_error_f("================ REASSEMBLY:");
phosg::fwritex(stderr, reassembly);
if (!assembled.data.empty()) {
phosg::log_error_f("================ BINDIFF:");
phosg::print_binary_diff(stderr, bin.data(), bin.size(), assembled.data.data(), assembled.data.size(), isatty(fileno(stderr)), 3, 0);
}
phosg::log_info_f("... {} {} {} ({}) SCRIPT FAILED", phosg::name_for_enum(vq->meta.version), name_for_language(vq->meta.language), vq->bin_filename(), vq->meta.name);
throw;
}
phosg::log_info_f("... {} {} {} ({}) SCRIPT OK", phosg::name_for_enum(vq->meta.version), name_for_language(vq->meta.language), vq->bin_filename(), vq->meta.name);
}
phosg::log_info_f("... {} {} {} ({}) OK", phosg::name_for_enum(vq->meta.version), name_for_language(vq->meta.language), vq->bin_filename(), vq->meta.name);
}
}
}
src/Map.cc
+130
View File
@@ -4127,6 +4127,136 @@ string MapFile::disassemble(bool reassembly, Version version) const {
return phosg::join(ret, "\n");
}
std::string MapFile::serialize() const {
using T = SectionHeader::Type;
phosg::StringWriter w;
auto write_section_header = [&](T type, uint32_t floor, uint32_t data_size) -> void {
uint32_t section_size = data_size + sizeof(SectionHeader);
w.put<SectionHeader>({static_cast<uint32_t>(type), section_size, floor, data_size});
};
// Try to serialize the sections in the same order as the input file; if the offsets are missing, serialize in
// (floor, type) order
struct SectionOrderEntry {
T type;
uint8_t floor;
size_t file_offset;
bool operator<(const SectionOrderEntry& other) const {
if (this->file_offset < other.file_offset) {
return true;
} else if (this->file_offset > other.file_offset) {
return false;
}
if (this->floor < other.floor) {
return true;
} else if (this->floor > other.floor) {
return false;
}
return static_cast<uint8_t>(this->type) < static_cast<uint8_t>(other.type);
}
};
std::set<SectionOrderEntry> sections_to_serialize;
for (uint8_t floor = 0; floor < this->sections_for_floor.size(); floor++) {
const auto& sf = this->sections_for_floor[floor];
if (sf.object_sets) {
sections_to_serialize.emplace(SectionOrderEntry{T::OBJECT_SETS, floor, sf.object_sets_file_offset});
}
if (sf.enemy_sets) {
sections_to_serialize.emplace(SectionOrderEntry{T::ENEMY_SETS, floor, sf.enemy_sets_file_offset});
}
if (sf.events1 || sf.events2) {
sections_to_serialize.emplace(SectionOrderEntry{T::EVENTS, floor, sf.events_file_offset});
}
if (sf.random_enemy_room_count && sf.random_enemy_locations) {
sections_to_serialize.emplace(SectionOrderEntry{
T::RANDOM_ENEMY_LOCATIONS, floor, sf.random_enemy_locations_file_offset});
}
if (sf.random_enemy_definitions && sf.random_enemy_weights) {
sections_to_serialize.emplace(SectionOrderEntry{
T::RANDOM_ENEMY_DEFINITIONS, floor, sf.random_enemy_definitions_file_offset});
}
}
for (const auto& section : sections_to_serialize) {
const auto& sf = this->sections_for_floor[section.floor];
switch (section.type) {
case T::OBJECT_SETS: {
size_t data_size = sizeof(ObjectSetEntry) * sf.object_set_count;
write_section_header(T::OBJECT_SETS, section.floor, data_size);
w.write(sf.object_sets, data_size);
break;
}
case T::ENEMY_SETS: {
size_t data_size = sizeof(EnemySetEntry) * sf.enemy_set_count;
write_section_header(T::ENEMY_SETS, section.floor, data_size);
w.write(sf.enemy_sets, data_size);
break;
}
case T::EVENTS: {
auto serialize_events_section = [&]<typename EventT>(const EventT* events) -> void {
EventsSectionHeader ev_header = {
.action_stream_offset = sizeof(EventsSectionHeader) + sizeof(EventT) * sf.event_count,
.entries_offset = sizeof(EventsSectionHeader),
.entry_count = sf.event_count,
.format = std::is_same_v<EventT, Event2Entry> ? 0x65767432 : 0x00000000,
};
size_t data_size = ev_header.action_stream_offset + sf.event_action_stream_bytes;
data_size = (data_size + 3) & (~3);
write_section_header(T::EVENTS, section.floor, data_size);
w.put(ev_header);
w.write(events, sf.event_count * sizeof(EventT));
w.write(sf.event_action_stream, sf.event_action_stream_bytes);
while (w.size() & 3) {
w.put_u8(0x00);
}
};
if (sf.events1) {
serialize_events_section(sf.events1);
} else if (sf.events2) {
serialize_events_section(sf.events2);
}
break;
}
case T::RANDOM_ENEMY_LOCATIONS: {
size_t data_size = sizeof(RandomEnemyLocationsHeader) +
sf.random_enemy_room_count * sizeof(RandomEnemyRoom) +
sf.random_enemy_location_count * sizeof(RandomEnemyLocation);
write_section_header(T::RANDOM_ENEMY_LOCATIONS, section.floor, data_size);
w.put(RandomEnemyLocationsHeader{
.room_table_offset = sizeof(RandomEnemyLocationsHeader),
.entries_offset = sizeof(RandomEnemyLocationsHeader) + sf.random_enemy_room_count * sizeof(RandomEnemyRoom),
.num_rooms = sf.random_enemy_room_count,
});
w.write(sf.random_enemy_rooms, sf.random_enemy_room_count * sizeof(RandomEnemyRoom));
w.write(sf.random_enemy_locations, sf.random_enemy_location_count * sizeof(RandomEnemyLocation));
break;
}
case T::RANDOM_ENEMY_DEFINITIONS: {
size_t data_size = sizeof(RandomEnemyDefinitionsHeader) +
sf.random_enemy_definition_count * sizeof(RandomEnemyDefinition) +
sf.random_enemy_weight_count * sizeof(RandomEnemyWeight);
write_section_header(T::RANDOM_ENEMY_DEFINITIONS, section.floor, data_size);
w.put(RandomEnemyDefinitionsHeader{
.entries_offset = sizeof(RandomEnemyDefinitionsHeader),
.weight_entries_offset = sizeof(RandomEnemyDefinitionsHeader) + sf.random_enemy_definition_count * sizeof(RandomEnemyDefinition),
.entry_count = sf.random_enemy_definition_count,
.weight_entry_count = sf.random_enemy_weight_count,
});
w.write(sf.random_enemy_definitions, sf.random_enemy_definition_count * sizeof(RandomEnemyDefinition));
w.write(sf.random_enemy_weights, sf.random_enemy_weight_count * sizeof(RandomEnemyWeight));
break;
}
default:
throw std::logic_error("invalid section type descriptor");
}
}
w.put<SectionHeader>({static_cast<uint32_t>(T::END), 0, 0, 0});
return std::move(w.str());
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Super map
src/Map.hh
+3 -1
View File
@@ -431,7 +431,7 @@ public:
// Quest constructor
MapFile(std::shared_ptr<const std::string> quest_data);
// Non-quest constructor
// Free-play constructor
MapFile(
uint8_t floor,
std::shared_ptr<const std::string> objects_data,
@@ -476,6 +476,8 @@ public:
static std::string disassemble_action_stream(const void* data, size_t size);
std::string disassemble(bool reassembly = false, Version version = Version::UNKNOWN) const;
std::string serialize() const;
protected:
static const std::array<uint32_t, 41> RAND_ENEMY_BASE_TYPES;
system/quests/download/q072-gc.dat
BIN
View File
Binary file not shown.
system/quests/events/q124-gc.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88101-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88102-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88103-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88104-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88105-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88106-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88107-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88108-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/c88109-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/d88201-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/d88202-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/d88203-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/d88204-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-maps-materialized/d88205-gc-00000000.dat
BIN
View File
Binary file not shown.
tests/challenge-mode-random.test.sh
Executable
+78
View File
@@ -0,0 +1,78 @@
#!/bin/sh
set -e
EXECUTABLE="$1"
if [ -z "$EXECUTABLE" ]; then
EXECUTABLE="./newserv"
fi
echo "... challenge-ep1/c88101-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88101-gc.dat ./tests/c88101-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88101-gc-00000000.dat ./tests/c88101-gc-00000000.dat
rm ./tests/c88101-gc-00000000.dat
echo "... challenge-ep1/c88102-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88102-gc.dat ./tests/c88102-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88102-gc-00000000.dat ./tests/c88102-gc-00000000.dat
rm ./tests/c88102-gc-00000000.dat
echo "... challenge-ep1/c88103-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88103-gc.dat ./tests/c88103-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88103-gc-00000000.dat ./tests/c88103-gc-00000000.dat
rm ./tests/c88103-gc-00000000.dat
echo "... challenge-ep1/c88104-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88104-gc.dat ./tests/c88104-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88104-gc-00000000.dat ./tests/c88104-gc-00000000.dat
rm ./tests/c88104-gc-00000000.dat
echo "... challenge-ep1/c88105-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88105-gc.dat ./tests/c88105-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88105-gc-00000000.dat ./tests/c88105-gc-00000000.dat
rm ./tests/c88105-gc-00000000.dat
echo "... challenge-ep1/c88106-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88106-gc.dat ./tests/c88106-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88106-gc-00000000.dat ./tests/c88106-gc-00000000.dat
rm ./tests/c88106-gc-00000000.dat
echo "... challenge-ep1/c88107-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88107-gc.dat ./tests/c88107-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88107-gc-00000000.dat ./tests/c88107-gc-00000000.dat
rm ./tests/c88107-gc-00000000.dat
echo "... challenge-ep1/c88108-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88108-gc.dat ./tests/c88108-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88108-gc-00000000.dat ./tests/c88108-gc-00000000.dat
rm ./tests/c88108-gc-00000000.dat
echo "... challenge-ep1/c88109-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep1/c88109-gc.dat ./tests/c88109-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/c88109-gc-00000000.dat ./tests/c88109-gc-00000000.dat
rm ./tests/c88109-gc-00000000.dat
echo "... challenge-ep2/d88201-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep2/d88201-gc.dat ./tests/d88201-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/d88201-gc-00000000.dat ./tests/d88201-gc-00000000.dat
rm ./tests/d88201-gc-00000000.dat
echo "... challenge-ep2/d88202-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep2/d88202-gc.dat ./tests/d88202-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/d88202-gc-00000000.dat ./tests/d88202-gc-00000000.dat
rm ./tests/d88202-gc-00000000.dat
echo "... challenge-ep2/d88203-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep2/d88203-gc.dat ./tests/d88203-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/d88203-gc-00000000.dat ./tests/d88203-gc-00000000.dat
rm ./tests/d88203-gc-00000000.dat
echo "... challenge-ep2/d88204-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep2/d88204-gc.dat ./tests/d88204-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/d88204-gc-00000000.dat ./tests/d88204-gc-00000000.dat
rm ./tests/d88204-gc-00000000.dat
echo "... challenge-ep2/d88205-gc.dat"
$EXECUTABLE materialize-map system/quests/challenge-ep2/d88205-gc.dat ./tests/d88205-gc-00000000.dat --seed=00000000
diff tests/challenge-maps-materialized/d88205-gc-00000000.dat ./tests/d88205-gc-00000000.dat
rm ./tests/d88205-gc-00000000.dat
tests/quest-compiler.test.sh
+1 -1
View File
@@ -7,4 +7,4 @@ if [ -z "$EXECUTABLE" ]; then
EXECUTABLE="./newserv"
fi
$EXECUTABLE check-quests --reassembly --config=tests/config.json
$EXECUTABLE check-quests --reassemble-scripts --reassemble-maps --config=tests/config.json