roles:
- [mon.a, mon.c, mgr.y, mds.a, osd.0, osd.1, osd.2, osd.3]
- [mon.b, mgr.x, mds.b, mds.c, osd.4, osd.5, osd.6, osd.7]
-- [client.0]
+- [client.0, client.1]
openstack:
- volumes: # attached to each instance
count: 4
roles:
- [mon.a, mon.c, mgr.y, mds.a, mds.b, mds.c, mds.d, osd.0, osd.1, osd.2, osd.3]
- [mon.b, mgr.x, mds.e, mds.f, mds.g, mds.h, mds.i, osd.4, osd.5, osd.6, osd.7]
-- [client.0]
+- [client.0, client.1]
openstack:
- volumes: # attached to each instance
count: 4
class TestExports(CephFSTestCase):
MDSS_REQUIRED = 2
+ CLIENTS_REQUIRED = 2
def _wait_subtrees(self, status, rank, test):
timeout = 30
self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/aa/bb', 0)])
self.mount_a.run_shell(["mv", "aa", "a/b/"])
self._wait_subtrees(status, 0, [('/1', 0), ('/1/4/5', 1), ('/1/2/3', 2), ('/a', 1), ('/a/b/aa/bb', 0)])
+
+ def test_session_race(self):
+ """
+ Test session creation race.
+
+ See: https://tracker.ceph.com/issues/24072#change-113056
+ """
+
+ self.fs.set_max_mds(2)
+ status = self.fs.wait_for_daemons()
+
+ rank1 = self.fs.get_rank(rank=1, status=status)
+ name1 = 'mds.'+rank1['name']
+
+ # Create a directory that is pre-exported to rank 1
+ self.mount_a.run_shell(["mkdir", "-p", "a/aa"])
+ self.mount_a.setfattr("a", "ceph.dir.pin", "1")
+ self._wait_subtrees(status, 1, [('/a', 1)])
+
+ # Now set the mds config to allow the race
+ self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "true"], rank=1)
+
+ # Now create another directory and try to export it
+ self.mount_b.run_shell(["mkdir", "-p", "b/bb"])
+ self.mount_b.setfattr("b", "ceph.dir.pin", "1")
+
+ time.sleep(5)
+
+ # Now turn off the race so that it doesn't wait again
+ self.fs.rank_asok(["config", "set", "mds_inject_migrator_session_race", "false"], rank=1)
+
+ # Now try to create a session with rank 1 by accessing a dir known to
+ # be there, if buggy, this should cause the rank 1 to crash:
+ self.mount_b.run_shell(["ls", "a"])
+
+ # Check if rank1 changed (standby tookover?)
+ new_rank1 = self.fs.get_rank(rank=1)
+ self.assertEqual(rank1['gid'], new_rank1['gid'])
Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
.set_default(0)
.set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),
+
+ Option("mds_inject_migrator_session_race", Option::TYPE_BOOL, Option::LEVEL_DEV)
+ .set_default(false),
});
}
"mds_max_purge_ops",
"mds_max_purge_ops_per_pg",
"mds_max_purge_files",
+ "mds_inject_migrator_session_race",
"clog_to_graylog",
"clog_to_graylog_host",
"clog_to_graylog_port",
dout(10) << " new session " << s << " for " << s->info.inst << " con " << con << dendl;
con->set_priv(s);
s->connection = con;
+ if (mds_rank) {
+ mds_rank->kick_waiters_for_any_client_connection();
+ }
} else {
dout(10) << " existing session " << s << " for " << s->info.inst << " existing con " << s->connection
<< ", new/authorizing con " << con << dendl;
void handle_conf_change(const struct md_config_t *conf,
const std::set <std::string> &changed)
{
+ mdcache->migrator->handle_conf_change(conf, changed, *mdsmap);
purge_queue.handle_conf_change(conf, changed, *mdsmap);
}
ceph_tid_t last_tid; // for mds-initiated requests (e.g. stray rename)
list<MDSInternalContextBase*> waiting_for_active, waiting_for_replay, waiting_for_reconnect, waiting_for_resolve;
+ list<MDSInternalContextBase*> waiting_for_any_client_connection;
list<MDSInternalContextBase*> replay_queue;
map<mds_rank_t, list<MDSInternalContextBase*> > waiting_for_active_peer;
map<epoch_t, list<MDSInternalContextBase*> > waiting_for_mdsmap;
waiting_for_active_peer[MDS_RANK_NONE].push_back(c);
}
+ void wait_for_any_client_connection(MDSInternalContextBase *c) {
+ waiting_for_any_client_connection.push_back(c);
+ }
+ void kick_waiters_for_any_client_connection(void) {
+ finish_contexts(g_ceph_context, waiting_for_any_client_connection);
+ }
void wait_for_active(MDSInternalContextBase *c) {
waiting_for_active.push_back(c);
}
#include "Mutation.h"
#include "include/filepath.h"
+#include "common/likely.h"
#include "events/EExport.h"
#include "events/EImportStart.h"
handle_export_prep(static_cast<MExportDirPrep*>(m));
break;
case MSG_MDS_EXPORTDIR:
- handle_export_dir(static_cast<MExportDir*>(m));
+ if (unlikely(inject_session_race)) {
+ dout(0) << "waiting for inject_session_race" << dendl;
+ mds->wait_for_any_client_connection(new C_MDS_RetryMessage(mds, m));
+ } else {
+ handle_export_dir(static_cast<MExportDir*>(m));
+ }
break;
case MSG_MDS_EXPORTDIRFINISH:
handle_export_finish(static_cast<MExportDirFinish*>(m));
mds->locker->eval(in, CEPH_CAP_LOCKS, true);
in->auth_unpin(this);
}
+
+void Migrator::handle_conf_change(const struct md_config_t *conf,
+ const std::set <std::string> &changed,
+ const MDSMap &mds_map)
+{
+ if (changed.count("mds_inject_migrator_session_race")) {
+ inject_session_race = conf->get_val<bool>("mds_inject_migrator_session_race");
+ dout(0) << "mds_inject_migrator_session_race is " << inject_session_race << dendl;
+ }
+}
}
// -- cons --
- Migrator(MDSRank *m, MDCache *c) : mds(m), cache(c) {}
-
+ Migrator(MDSRank *m, MDCache *c) : mds(m), cache(c) {
+ inject_session_race = g_conf->get_val<bool>("mds_inject_migrator_session_race");
+ }
+ void handle_conf_change(const struct md_config_t *conf,
+ const std::set <std::string> &changed,
+ const MDSMap &mds_map);
protected:
// export fun
private:
MDSRank *mds;
MDCache *cache;
+ bool inject_session_race = false;
};
#endif
projects / ceph.git / commitdiff