MEMPOOL_DEFINE_OBJECT_FACTORY(PG::CephPeeringEvt, pg_peering_evt, osd);
+void PGStateHistory::enter(PG* pg, const utime_t entime, const char* state)
+{
+ // Ignore trimming state machine for now
+ if (::strstr(state, "Trimming") != NULL) {
+ return;
+ } else if (pi != nullptr) {
+ pi->enter_state(entime, state);
+ } else {
+ // Store current state since we can't reliably take the PG lock here
+ if ( tmppi == nullptr) {
+ tmppi = std::unique_ptr<PGStateInstance>(new PGStateInstance);
+ }
+
+ thispg = pg;
+ tmppi->enter_state(entime, state);
+ }
+}
+
+void PGStateHistory::exit(const char* state) {
+ // Ignore trimming state machine for now
+ // Do nothing if PG is being destroyed!
+ if (::strstr(state, "Trimming") != NULL || pg_in_destructor) {
+ return;
+ } else {
+ bool ilocked = false;
+ if(!thispg->is_locked()) {
+ thispg->lock();
+ ilocked = true;
+ }
+ if (pi == nullptr) {
+ buffer.push_back(std::unique_ptr<PGStateInstance>(tmppi.release()));
+ pi = buffer.back().get();
+ pi->setepoch(thispg->get_osdmap()->get_epoch());
+ }
+
+ pi->exit_state(ceph_clock_now());
+ if (::strcmp(state, "Reset") == 0) {
+ this->reset();
+ }
+ if(ilocked) {
+ thispg->unlock();
+ }
+ }
+}
+
+void PGStateHistory::dump(Formatter* f) const {
+ f->open_array_section("history");
+ for (auto pi = buffer.begin(); pi != buffer.end(); ++pi) {
+ f->open_object_section("states");
+ f->dump_stream("epoch") << (*pi)->this_epoch;
+ for (auto she : (*pi)->state_history) {
+ f->dump_string("state", std::get<2>(she));
+ f->dump_stream("enter") << std::get<0>(she);
+ f->dump_stream("exit") << std::get<1>(she);
+ }
+ f->close_section();
+ }
+ f->close_section();
+}
+
void PG::get(const char* tag)
{
ref++;
PG::~PG()
{
+ pgstate_history.set_pg_in_destructor();
#ifdef PG_DEBUG_REFS
osd->remove_pgid(info.pgid, this);
#endif
/*------Crashed-------*/
PG::RecoveryState::Crashed::Crashed(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Crashed")
+ NamedState(context< RecoveryMachine >().pg, "Crashed")
{
context< RecoveryMachine >().log_enter(state_name);
assert(0 == "we got a bad state machine event");
/*------Initial-------*/
PG::RecoveryState::Initial::Initial(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Initial")
+ NamedState(context< RecoveryMachine >().pg, "Initial")
{
context< RecoveryMachine >().log_enter(state_name);
}
/*------Started-------*/
PG::RecoveryState::Started::Started(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started")
+ NamedState(context< RecoveryMachine >().pg, "Started")
{
context< RecoveryMachine >().log_enter(state_name);
}
/*--------Reset---------*/
PG::RecoveryState::Reset::Reset(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Reset")
+ NamedState(context< RecoveryMachine >().pg, "Reset")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*-------Start---------*/
PG::RecoveryState::Start::Start(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Start")
+ NamedState(context< RecoveryMachine >().pg, "Start")
{
context< RecoveryMachine >().log_enter(state_name);
/*---------Primary--------*/
PG::RecoveryState::Primary::Primary(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*---------Peering--------*/
PG::RecoveryState::Peering::Peering(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering"),
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Peering"),
history_les_bound(false)
{
context< RecoveryMachine >().log_enter(state_name);
/*------Backfilling-------*/
PG::RecoveryState::Backfilling::Backfilling(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Backfilling")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/Backfilling")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
PG::RecoveryState::WaitRemoteBackfillReserved::WaitRemoteBackfillReserved(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteBackfillReserved"),
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/WaitRemoteBackfillReserved"),
backfill_osd_it(context< Active >().remote_shards_to_reserve_backfill.begin())
{
context< RecoveryMachine >().log_enter(state_name);
/*--WaitLocalBackfillReserved--*/
PG::RecoveryState::WaitLocalBackfillReserved::WaitLocalBackfillReserved(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitLocalBackfillReserved")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/WaitLocalBackfillReserved")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*----NotBackfilling------*/
PG::RecoveryState::NotBackfilling::NotBackfilling(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/NotBackfilling")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/NotBackfilling")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*----NotRecovering------*/
PG::RecoveryState::NotRecovering::NotRecovering(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/NotRecovering")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/NotRecovering")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*---RepNotRecovering----*/
PG::RecoveryState::RepNotRecovering::RepNotRecovering(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepNotRecovering")
+ NamedState(context< RecoveryMachine >().pg, "Started/ReplicaActive/RepNotRecovering")
{
context< RecoveryMachine >().log_enter(state_name);
}
/*---RepWaitRecoveryReserved--*/
PG::RecoveryState::RepWaitRecoveryReserved::RepWaitRecoveryReserved(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepWaitRecoveryReserved")
+ NamedState(context< RecoveryMachine >().pg, "Started/ReplicaActive/RepWaitRecoveryReserved")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*-RepWaitBackfillReserved*/
PG::RecoveryState::RepWaitBackfillReserved::RepWaitBackfillReserved(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepWaitBackfillReserved")
+ NamedState(context< RecoveryMachine >().pg, "Started/ReplicaActive/RepWaitBackfillReserved")
{
context< RecoveryMachine >().log_enter(state_name);
}
/*---RepRecovering-------*/
PG::RecoveryState::RepRecovering::RepRecovering(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive/RepRecovering")
+ NamedState(context< RecoveryMachine >().pg, "Started/ReplicaActive/RepRecovering")
{
context< RecoveryMachine >().log_enter(state_name);
}
/*------Activating--------*/
PG::RecoveryState::Activating::Activating(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Activating")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/Activating")
{
context< RecoveryMachine >().log_enter(state_name);
}
PG::RecoveryState::WaitLocalRecoveryReserved::WaitLocalRecoveryReserved(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitLocalRecoveryReserved")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/WaitLocalRecoveryReserved")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
PG::RecoveryState::WaitRemoteRecoveryReserved::WaitRemoteRecoveryReserved(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/WaitRemoteRecoveryReserved"),
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/WaitRemoteRecoveryReserved"),
remote_recovery_reservation_it(context< Active >().remote_shards_to_reserve_recovery.begin())
{
context< RecoveryMachine >().log_enter(state_name);
PG::RecoveryState::Recovering::Recovering(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Recovering")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/Recovering")
{
context< RecoveryMachine >().log_enter(state_name);
PG::RecoveryState::Recovered::Recovered(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Recovered")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/Recovered")
{
pg_shard_t auth_log_shard;
PG::RecoveryState::Clean::Clean(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active/Clean")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active/Clean")
{
context< RecoveryMachine >().log_enter(state_name);
/*---------Active---------*/
PG::RecoveryState::Active::Active(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Active"),
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Active"),
remote_shards_to_reserve_recovery(
unique_osd_shard_set(
context< RecoveryMachine >().pg->pg_whoami,
/*------ReplicaActive-----*/
PG::RecoveryState::ReplicaActive::ReplicaActive(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/ReplicaActive")
+ NamedState(context< RecoveryMachine >().pg, "Started/ReplicaActive")
{
context< RecoveryMachine >().log_enter(state_name);
/*-------Stray---*/
PG::RecoveryState::Stray::Stray(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Stray")
+ NamedState(context< RecoveryMachine >().pg, "Started/Stray")
{
context< RecoveryMachine >().log_enter(state_name);
/*--------GetInfo---------*/
PG::RecoveryState::GetInfo::GetInfo(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetInfo")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Peering/GetInfo")
{
context< RecoveryMachine >().log_enter(state_name);
PG::RecoveryState::GetLog::GetLog(my_context ctx)
: my_base(ctx),
NamedState(
- context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetLog"),
+ context< RecoveryMachine >().pg, "Started/Primary/Peering/GetLog"),
msg(0)
{
context< RecoveryMachine >().log_enter(state_name);
/*------WaitActingChange--------*/
PG::RecoveryState::WaitActingChange::WaitActingChange(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitActingChange")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Peering/WaitActingChange")
{
context< RecoveryMachine >().log_enter(state_name);
}
/*------Down--------*/
PG::RecoveryState::Down::Down(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/Down")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Peering/Down")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*------Incomplete--------*/
PG::RecoveryState::Incomplete::Incomplete(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/Incomplete")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Peering/Incomplete")
{
context< RecoveryMachine >().log_enter(state_name);
PG *pg = context< RecoveryMachine >().pg;
/*------GetMissing--------*/
PG::RecoveryState::GetMissing::GetMissing(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/GetMissing")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Peering/GetMissing")
{
context< RecoveryMachine >().log_enter(state_name);
/*------WaitUpThru--------*/
PG::RecoveryState::WaitUpThru::WaitUpThru(my_context ctx)
: my_base(ctx),
- NamedState(context< RecoveryMachine >().pg->cct, "Started/Primary/Peering/WaitUpThru")
+ NamedState(context< RecoveryMachine >().pg, "Started/Primary/Peering/WaitUpThru")
{
context< RecoveryMachine >().log_enter(state_name);
}
#include <boost/statechart/transition.hpp>
#include <boost/statechart/event_base.hpp>
#include <boost/scoped_ptr.hpp>
+#include <boost/circular_buffer.hpp>
#include "include/memory.h"
#include "include/mempool.h"
#include <atomic>
#include <list>
#include <memory>
+#include <stack>
#include <string>
+#include <tuple>
using namespace std;
// #include "include/unordered_map.h"
// #include "include/unordered_set.h"
-
//#define DEBUG_RECOVERY_OIDS // track set of recovering oids explicitly, to find counting bugs
class OSD;
void intrusive_ptr_add_ref(PG *pg);
void intrusive_ptr_release(PG *pg);
+using state_history_entry = std::tuple<utime_t, utime_t, const char*>;
+using embedded_state = std::pair<utime_t, const char*>;
+
+struct PGStateInstance {
+ // Time spent in pg states
+
+ void setepoch(const epoch_t current_epoch) {
+ this_epoch = current_epoch;
+ }
+
+ void enter_state(const utime_t entime, const char* state) {
+ embedded_states.push(std::make_pair(entime, state));
+ }
+
+ void exit_state(const utime_t extime) {
+ embedded_state this_state = embedded_states.top();
+ state_history.push_back(state_history_entry{
+ this_state.first, extime, this_state.second});
+ embedded_states.pop();
+ }
+
+ epoch_t this_epoch;
+ utime_t enter_time;
+ std::vector<state_history_entry> state_history;
+ std::stack<embedded_state> embedded_states;
+};
+
+class PGStateHistory {
+ // Member access protected with the PG lock
+public:
+ PGStateHistory() : buffer(10) {}
+
+ void enter(PG* pg, const utime_t entime, const char* state);
+
+ void exit(const char* state);
+
+ void reset() {
+ pi = nullptr;
+ }
+
+ void set_pg_in_destructor() { pg_in_destructor = true; }
+
+ void dump(Formatter* f) const;
+
+private:
+ bool pg_in_destructor = false;
+ PG* thispg = nullptr;
+ std::unique_ptr<PGStateInstance> tmppi;
+ PGStateInstance* pi = nullptr;
+ boost::circular_buffer<std::unique_ptr<PGStateInstance>> buffer;
+
+};
+
#ifdef PG_DEBUG_REFS
#include "common/tracked_int_ptr.hpp"
uint64_t get_with_id(PG *pg);
}
};
+
+ PGStateHistory pgstate_history;
+
struct NamedState {
const char *state_name;
utime_t enter_time;
+ PG* pg;
const char *get_state_name() { return state_name; }
- NamedState(CephContext *cct_, const char *state_name_)
- : state_name(state_name_),
- enter_time(ceph_clock_now()) {}
- virtual ~NamedState() {}
+ NamedState(PG *pg_, const char *state_name_)
+ : state_name(state_name_), enter_time(ceph_clock_now()), pg(pg_) {
+ pg->pgstate_history.enter(pg, enter_time, state_name);
+ }
+ virtual ~NamedState() { pg->pgstate_history.exit(state_name); }
};