diff --git a/src/Lobby.cc b/src/Lobby.cc
index 8828ee2d..904854f9 100644
--- a/src/Lobby.cc
+++ b/src/Lobby.cc
@@ -274,24 +274,16 @@ void Lobby::load_maps() {
   auto rare_rates = this->rare_enemy_rates ? this->rare_enemy_rates : MapState::DEFAULT_RARE_ENEMIES;
 
   if (this->quest) {
+    this->log.info_f("Loading quest supermap");
+    auto supermap = this->quest->get_supermap(this->random_seed);
     this->map_state = make_shared<MapState>(
-        this->lobby_id,
-        this->difficulty,
-        this->event,
-        this->random_seed,
-        this->rare_enemy_rates,
-        this->rand_crypt,
-        this->quest->get_supermap(this->random_seed));
+        this->lobby_id, this->difficulty, this->event, this->random_seed, this->rare_enemy_rates, this->rand_crypt, supermap);
   } else {
+    this->log.info_f("Loading free play supermaps");
     auto s = this->require_server_state();
+    auto supermaps = s->supermaps_for_variations(this->episode, this->mode, this->difficulty, this->variations);
     this->map_state = make_shared<MapState>(
-        this->lobby_id,
-        this->difficulty,
-        this->event,
-        this->random_seed,
-        this->rare_enemy_rates,
-        this->rand_crypt,
-        s->supermaps_for_variations(this->episode, this->mode, this->difficulty, this->variations));
+        this->lobby_id, this->difficulty, this->event, this->random_seed, this->rare_enemy_rates, this->rand_crypt, supermaps);
   }
 
   if (this->check_flag(Lobby::Flag::DEBUG)) {
@@ -749,7 +741,7 @@ void Lobby::assign_inventory_and_bank_item_ids(shared_ptr<Client> c, bool consum
 
   if (c->log.info_f("Assigned inventory item IDs{}", consume_ids ? "" : " but did not mark IDs as used")) {
     c->print_inventory();
-    if (c->version() == Version::BB_V4) {
+    if ((c->version() == Version::BB_V4) && !c->has_overlay()) {
       auto bank = c->bank_file();
       if (!bank->items.empty()) {
         bank->assign_ids(0x99000000 + (c->lobby_client_id << 20));
diff --git a/src/ReceiveCommands.cc b/src/ReceiveCommands.cc
index 697e465a..d981966e 100644
--- a/src/ReceiveCommands.cc
+++ b/src/ReceiveCommands.cc
@@ -2370,6 +2370,10 @@ static void on_quest_loaded(shared_ptr<Lobby> l) {
   if (!l->quest) {
     throw logic_error("on_quest_loaded called without a quest loaded");
   }
+  auto leader_c = l->clients.at(l->leader_id);
+  if (!leader_c) {
+    throw std::logic_error("lobby has no leader");
+  }
 
   // Replace the free-play map with the quest's map
   l->load_maps();
@@ -2391,18 +2395,24 @@ static void on_quest_loaded(shared_ptr<Lobby> l) {
     }
 
     lc->delete_overlay();
-    if (l->quest->battle_rules) {
-      lc->change_bank(lc->bb_character_index);
-      lc->create_battle_overlay(l->quest->battle_rules, s->level_table(lc->version()));
-      lc->log.info_f("Created battle overlay");
-    } else if (l->quest->challenge_template_index >= 0 && !is_v4(lc->version())) {
-      // On BB, the client will send a sequence of DF commands that creates the
-      // overlay; on non-BB, we do it at quest start time instead (hence the
-      // version check above).
-      lc->change_bank(lc->bb_character_index);
+
+    if ((l->quest->challenge_template_index >= 0) && !is_v4(leader_c->version())) {
+      // If the leader is BB, they will send an 02DF command that will create
+      // the overlays later; on other versions, we do it at quest start time
+      // (now) instead, hence the version check above.
+      if (is_v4(lc->version())) {
+        lc->change_bank(lc->bb_character_index);
+      }
       lc->create_challenge_overlay(lc->version(), l->quest->challenge_template_index, s->level_table(lc->version()));
       lc->log.info_f("Created challenge overlay");
       l->assign_inventory_and_bank_item_ids(lc, true);
+
+    } else if (l->quest->battle_rules) {
+      if (is_v4(lc->version())) {
+        lc->change_bank(lc->bb_character_index);
+      }
+      lc->create_battle_overlay(l->quest->battle_rules, s->level_table(lc->version()));
+      lc->log.info_f("Created battle overlay");
     }
   }
 }
@@ -4064,6 +4074,13 @@ static asio::awaitable<void> on_DF_BB(shared_ptr<Client> c, Channel::Message& ms
       if (!l->quest) {
         throw runtime_error("challenge mode character template config command sent in non-challenge game");
       }
+      auto leader_c = l->clients.at(l->leader_id);
+      if (!leader_c) {
+        throw logic_error("lobby has no leader");
+      }
+      if (leader_c != c) {
+        throw runtime_error("non-leader sent 02DF command");
+      }
       auto vq = l->quest->version(Version::BB_V4, c->language());
       if (vq->challenge_template_index != static_cast<ssize_t>(cmd.template_index)) {
         throw runtime_error("challenge template index in quest metadata does not match index sent by client");
@@ -4074,10 +4091,13 @@ static asio::awaitable<void> on_DF_BB(shared_ptr<Client> c, Channel::Message& ms
       }
 
       for (auto lc : l->clients) {
-        // On non-BB, there is no DF command, and overlays are created at quest
-        // start time instead, hence the version check here.
-        if (lc && is_v4(lc->version())) {
-          lc->change_bank(lc->bb_character_index);
+        // See comment in on_quest_loaded about when the leader is responsible
+        // for creating challenge overlays vs. when the server should do it at
+        // quest load time
+        if (lc) {
+          if (is_v4(lc->version())) {
+            lc->change_bank(lc->bb_character_index);
+          }
           lc->create_challenge_overlay(lc->version(), l->quest->challenge_template_index, s->level_table(lc->version()));
           lc->log.info_f("Created challenge overlay");
           l->assign_inventory_and_bank_item_ids(lc, true);
diff --git a/src/ReceiveSubcommands.cc b/src/ReceiveSubcommands.cc
index ed5a52ac..fe6cd9d4 100644
--- a/src/ReceiveSubcommands.cc
+++ b/src/ReceiveSubcommands.cc
@@ -4436,7 +4436,9 @@ static asio::awaitable<void> on_battle_restart_bb(shared_ptr<Client> c, Subcomma
   for (auto& lc : l->clients) {
     if (lc) {
       lc->delete_overlay();
-      lc->change_bank(lc->bb_character_index);
+      if (is_v4(lc->version())) {
+        lc->change_bank(lc->bb_character_index);
+      }
       lc->create_battle_overlay(new_rules, s->level_table(c->version()));
     }
   }
@@ -4513,6 +4515,11 @@ static asio::awaitable<void> on_challenge_mode_retry_or_quit(shared_ptr<Client>
   const auto& cmd = msg.check_size_t<G_SelectChallengeModeFailureOption_6x97>();
 
   auto l = c->require_lobby();
+  auto leader_c = l->clients.at(l->leader_id);
+  if (leader_c != c) {
+    throw runtime_error("6x97 sent by non-leader");
+  }
+
   if (l->is_game() && (cmd.is_retry == 1) && l->quest && (l->quest->challenge_template_index >= 0)) {
     auto s = l->require_server_state();
 
@@ -4520,12 +4527,18 @@ static asio::awaitable<void> on_challenge_mode_retry_or_quit(shared_ptr<Client>
       m.clear();
     }
 
-    for (auto lc : l->clients) {
-      if (lc) {
-        lc->change_bank(lc->bb_character_index);
-        lc->create_challenge_overlay(lc->version(), l->quest->challenge_template_index, s->level_table(c->version()));
-        lc->log.info_f("Created challenge overlay");
-        l->assign_inventory_and_bank_item_ids(lc, true);
+    // If the leader (c) is BB, they are expected to send 02DF later, which
+    // will recreate the overlays.
+    if (!is_v4(c->version())) {
+      for (auto lc : l->clients) {
+        if (lc) {
+          if (is_v4(lc->version())) {
+            lc->change_bank(lc->bb_character_index);
+          }
+          lc->create_challenge_overlay(lc->version(), l->quest->challenge_template_index, s->level_table(c->version()));
+          lc->log.info_f("Created challenge overlay");
+          l->assign_inventory_and_bank_item_ids(lc, true);
+        }
       }
     }