issued |= cap.issued;
flags |= CEPH_CAP_FLAG_AUTH;
}
+ } else {
+ inc_pinned_icaps();
}
check_cap_issue(in, issued);
cap->issue_seq,
cap->mseq,
cap_epoch_barrier);
+ } else {
+ dec_pinned_icaps();
}
+
if (in.auth_cap == cap) {
if (in.flushing_cap_item.is_on_list()) {
ldout(cct, 10) << " removing myself from flushing_cap list" << dendl;
void Client::flush_cap_releases()
{
+ uint64_t nr_caps = 0;
+
// send any cap releases
for (auto &p : mds_sessions) {
auto &session = p.second;
if (session.release && mdsmap->is_clientreplay_or_active_or_stopping(
p.first)) {
+ nr_caps += session.release->caps.size();
if (cct->_conf->client_inject_release_failure) {
ldout(cct, 20) << __func__ << " injecting failure to send cap release message" << dendl;
} else {
session.release.reset();
}
}
+
+ if (nr_caps > 0) {
+ dec_pinned_icaps(nr_caps);
+ }
}
void Client::renew_and_flush_cap_releases()
metric = ClientMetricMessage(DentryLeasePayload(dlease_hits, dlease_misses, nr));
message.push_back(metric);
+ // opened files
+ {
+ auto [opened_files, total_inodes] = get_opened_files_rates();
+ metric = ClientMetricMessage(OpenedFilesPayload(opened_files, total_inodes));
+ }
+ message.push_back(metric);
+
+ // pinned i_caps
+ {
+ auto [pinned_icaps, total_inodes] = get_pinned_icaps_rates();
+ metric = ClientMetricMessage(PinnedIcapsPayload(pinned_icaps, total_inodes));
+ }
+ message.push_back(metric);
+
+ // opened inodes
+ {
+ auto [opened_inodes, total_inodes] = get_opened_inodes_rates();
+ metric = ClientMetricMessage(OpenedInodesPayload(opened_inodes, total_inodes));
+ }
+ message.push_back(metric);
+
session->con->send_message2(make_message<MClientMetrics>(std::move(message)));
}
return std::make_pair(cap_hits, cap_misses);
}
+ void inc_opened_files() {
+ ++opened_files;
+ }
+ void dec_opened_files() {
+ --opened_files;
+ }
+ std::pair<uint64_t, uint64_t> get_opened_files_rates() {
+ return std::make_pair(opened_files, inode_map.size());
+ }
+
+ void inc_pinned_icaps() {
+ ++pinned_icaps;
+ }
+ void dec_pinned_icaps(uint64_t nr=1) {
+ pinned_icaps -= nr;
+ }
+ std::pair<uint64_t, uint64_t> get_pinned_icaps_rates() {
+ return std::make_pair(pinned_icaps, inode_map.size());
+ }
+
+ void inc_opened_inodes() {
+ ++opened_inodes;
+ }
+ void dec_opened_inodes() {
+ --opened_inodes;
+ }
+ std::pair<uint64_t, uint64_t> get_opened_inodes_rates() {
+ return std::make_pair(opened_inodes, inode_map.size());
+ }
+
xlist<Inode*> &get_dirty_list() { return dirty_list; }
/* timer_lock for 'timer' */
uint64_t cap_hits = 0;
uint64_t cap_misses = 0;
+ uint64_t opened_files = 0;
+ uint64_t pinned_icaps = 0;
+ uint64_t opened_inodes = 0;
+
ceph::spinlock delay_i_lock;
std::map<Inode*,int> delay_i_release;
};
void Inode::get_open_ref(int mode)
{
+ client->inc_opened_files();
+ if (open_by_mode.count(mode) == 0)
+ client->inc_opened_inodes();
open_by_mode[mode]++;
break_deleg(!(mode & CEPH_FILE_MODE_WR));
}
//cout << "open_by_mode[" << mode << "] " << open_by_mode[mode] << " -> " << (open_by_mode[mode]-1) << std::endl;
auto& ref = open_by_mode.at(mode);
ceph_assert(ref > 0);
- if (--ref == 0)
+ client->dec_opened_files();
+ if (--ref == 0) {
+ client->dec_opened_inodes();
return true;
+ }
return false;
}
CLIENT_METRIC_TYPE_WRITE_LATENCY,
CLIENT_METRIC_TYPE_METADATA_LATENCY,
CLIENT_METRIC_TYPE_DENTRY_LEASE,
+ CLIENT_METRIC_TYPE_OPENED_FILES,
+ CLIENT_METRIC_TYPE_PINNED_ICAPS,
+ CLIENT_METRIC_TYPE_OPENED_INODES,
};
inline std::ostream &operator<<(std::ostream &os, const ClientMetricType &type) {
switch(type) {
case ClientMetricType::CLIENT_METRIC_TYPE_DENTRY_LEASE:
os << "DENTRY_LEASE";
break;
+ case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES:
+ os << "OPENED_FILES";
+ break;
+ case ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS:
+ os << "PINNED_ICAPS";
+ break;
+ case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES:
+ os << "OPENED_INODES";
+ break;
default:
ceph_abort();
}
}
};
+struct OpenedFilesPayload {
+ static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES;
+
+ uint64_t opened_files = 0;
+ uint64_t total_inodes = 0;
+
+ OpenedFilesPayload() { }
+ OpenedFilesPayload(uint64_t opened_files, uint64_t total_inodes)
+ : opened_files(opened_files), total_inodes(total_inodes) {
+ }
+
+ void encode(bufferlist &bl) const {
+ using ceph::encode;
+ ENCODE_START(1, 1, bl);
+ encode(opened_files, bl);
+ encode(total_inodes, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator &iter) {
+ using ceph::decode;
+ DECODE_START(1, iter);
+ decode(opened_files, iter);
+ decode(total_inodes, iter);
+ DECODE_FINISH(iter);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_int("opened_files", opened_files);
+ f->dump_int("total_inodes", total_inodes);
+ }
+
+ void print(ostream *out) const {
+ *out << "opened_files: " << opened_files << " "
+ << "total_inodes: " << total_inodes;
+ }
+};
+
+struct PinnedIcapsPayload {
+ static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS;
+
+ uint64_t pinned_icaps = 0;
+ uint64_t total_inodes = 0;
+
+ PinnedIcapsPayload() { }
+ PinnedIcapsPayload(uint64_t pinned_icaps, uint64_t total_inodes)
+ : pinned_icaps(pinned_icaps), total_inodes(total_inodes) {
+ }
+
+ void encode(bufferlist &bl) const {
+ using ceph::encode;
+ ENCODE_START(1, 1, bl);
+ encode(pinned_icaps, bl);
+ encode(total_inodes, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator &iter) {
+ using ceph::decode;
+ DECODE_START(1, iter);
+ decode(pinned_icaps, iter);
+ decode(total_inodes, iter);
+ DECODE_FINISH(iter);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_int("pinned_icaps", pinned_icaps);
+ f->dump_int("total_inodes", total_inodes);
+ }
+
+ void print(ostream *out) const {
+ *out << "pinned_icaps: " << pinned_icaps << " "
+ << "total_inodes: " << total_inodes;
+ }
+};
+
+struct OpenedInodesPayload {
+ static const ClientMetricType METRIC_TYPE = ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES;
+
+ uint64_t opened_inodes = 0;
+ uint64_t total_inodes = 0;
+
+ OpenedInodesPayload() { }
+ OpenedInodesPayload(uint64_t opened_inodes, uint64_t total_inodes)
+ : opened_inodes(opened_inodes), total_inodes(total_inodes) {
+ }
+
+ void encode(bufferlist &bl) const {
+ using ceph::encode;
+ ENCODE_START(1, 1, bl);
+ encode(opened_inodes, bl);
+ encode(total_inodes, bl);
+ ENCODE_FINISH(bl);
+ }
+
+ void decode(bufferlist::const_iterator &iter) {
+ using ceph::decode;
+ DECODE_START(1, iter);
+ decode(opened_inodes, iter);
+ decode(total_inodes, iter);
+ DECODE_FINISH(iter);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_int("opened_inodes", opened_inodes);
+ f->dump_int("total_inodes", total_inodes);
+ }
+
+ void print(ostream *out) const {
+ *out << "opened_inodes: " << opened_inodes << " "
+ << "total_inodes: " << total_inodes;
+ }
+};
+
struct UnknownPayload {
static const ClientMetricType METRIC_TYPE = static_cast<ClientMetricType>(-1);
WriteLatencyPayload,
MetadataLatencyPayload,
DentryLeasePayload,
+ OpenedFilesPayload,
+ PinnedIcapsPayload,
+ OpenedInodesPayload,
UnknownPayload> ClientMetricPayload;
// metric update message sent by clients
case ClientMetricType::CLIENT_METRIC_TYPE_DENTRY_LEASE:
payload = DentryLeasePayload();
break;
+ case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_FILES:
+ payload = OpenedFilesPayload();
+ break;
+ case ClientMetricType::CLIENT_METRIC_TYPE_PINNED_ICAPS:
+ payload = PinnedIcapsPayload();
+ break;
+ case ClientMetricType::CLIENT_METRIC_TYPE_OPENED_INODES:
+ payload = OpenedInodesPayload();
+ break;
default:
payload = UnknownPayload();
break;
}
};
+struct OpenedFilesMetric {
+ uint64_t opened_files = 0;
+ uint64_t total_inodes = 0;
+ bool updated = false;
+
+ DENC(OpenedFilesMetric, v, p) {
+ DENC_START(1, 1, p);
+ denc(v.opened_files, p);
+ denc(v.total_inodes, p);
+ denc(v.updated, p);
+ DENC_FINISH(p);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_unsigned("opened_files", opened_files);
+ f->dump_unsigned("total_inodes", total_inodes);
+ }
+
+ friend std::ostream& operator<<(std::ostream& os, const OpenedFilesMetric &metric) {
+ os << "{opened_files=" << metric.opened_files << ", total_inodes="
+ << metric.total_inodes << "}";
+ return os;
+ }
+};
+
+struct PinnedIcapsMetric {
+ uint64_t pinned_icaps = 0;
+ uint64_t total_inodes = 0;
+ bool updated = false;
+
+ DENC(PinnedIcapsMetric, v, p) {
+ DENC_START(1, 1, p);
+ denc(v.pinned_icaps, p);
+ denc(v.total_inodes, p);
+ denc(v.updated, p);
+ DENC_FINISH(p);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_unsigned("pinned_icaps", pinned_icaps);
+ f->dump_unsigned("total_inodes", total_inodes);
+ }
+
+ friend std::ostream& operator<<(std::ostream& os, const PinnedIcapsMetric &metric) {
+ os << "{pinned_icaps=" << metric.pinned_icaps << ", total_inodes="
+ << metric.total_inodes << "}";
+ return os;
+ }
+};
+
+struct OpenedInodesMetric {
+ uint64_t opened_inodes = 0;
+ uint64_t total_inodes = 0;
+ bool updated = false;
+
+ DENC(OpenedInodesMetric, v, p) {
+ DENC_START(1, 1, p);
+ denc(v.opened_inodes, p);
+ denc(v.total_inodes, p);
+ denc(v.updated, p);
+ DENC_FINISH(p);
+ }
+
+ void dump(Formatter *f) const {
+ f->dump_unsigned("opened_inodes", opened_inodes);
+ f->dump_unsigned("total_inodes", total_inodes);
+ }
+
+ friend std::ostream& operator<<(std::ostream& os, const OpenedInodesMetric &metric) {
+ os << "{opened_inodes=" << metric.opened_inodes << ", total_inodes="
+ << metric.total_inodes << "}";
+ return os;
+ }
+};
+
WRITE_CLASS_DENC(CapHitMetric)
WRITE_CLASS_DENC(ReadLatencyMetric)
WRITE_CLASS_DENC(WriteLatencyMetric)
WRITE_CLASS_DENC(MetadataLatencyMetric)
WRITE_CLASS_DENC(DentryLeaseHitMetric)
+WRITE_CLASS_DENC(OpenedFilesMetric)
+WRITE_CLASS_DENC(PinnedIcapsMetric)
+WRITE_CLASS_DENC(OpenedInodesMetric)
// metrics that are forwarded to the MDS by client(s).
struct Metrics {
WriteLatencyMetric write_latency_metric;
MetadataLatencyMetric metadata_latency_metric;
DentryLeaseHitMetric dentry_lease_metric;
+ OpenedFilesMetric opened_files_metric;
+ PinnedIcapsMetric pinned_icaps_metric;
+ OpenedInodesMetric opened_inodes_metric;
// metric update type
uint32_t update_type = UpdateType::UPDATE_TYPE_REFRESH;
DENC(Metrics, v, p) {
- DENC_START(2, 1, p);
+ DENC_START(3, 1, p);
denc(v.update_type, p);
denc(v.cap_hit_metric, p);
denc(v.read_latency_metric, p);
if (struct_v >= 2) {
denc(v.dentry_lease_metric, p);
}
+ if (struct_v >= 3) {
+ denc(v.opened_files_metric, p);
+ denc(v.pinned_icaps_metric, p);
+ denc(v.opened_inodes_metric, p);
+ }
DENC_FINISH(p);
}
f->dump_object("write_latency_metric", write_latency_metric);
f->dump_object("metadata_latency_metric", metadata_latency_metric);
f->dump_object("dentry_lease_metric", dentry_lease_metric);
+ f->dump_object("opened_files_metric", opened_files_metric);
+ f->dump_object("pinned_icaps_metric", pinned_icaps_metric);
+ f->dump_object("opened_inodes_metric", opened_inodes_metric);
}
friend std::ostream& operator<<(std::ostream& os, const Metrics& metrics) {
<< ", write_latency=" << metrics.write_latency_metric
<< ", metadata_latency=" << metrics.metadata_latency_metric
<< ", dentry_lease =" << metrics.dentry_lease_metric
+ << ", opened_files_metric =" << metrics.opened_files_metric
+ << ", pinned_icaps_metric =" << metrics.pinned_icaps_metric
+ << ", opened_inodes_metric =" << metrics.opened_inodes_metric
<< "}]";
return os;
}
c->second = metrics.dentry_lease_metric.misses;
}
break;
+ case MDSPerformanceCounterType::OPENED_FILES_METRIC:
+ if (metrics.opened_files_metric.updated) {
+ c->first = metrics.opened_files_metric.opened_files;
+ c->second = metrics.opened_files_metric.total_inodes;
+ }
+ break;
+ case MDSPerformanceCounterType::PINNED_ICAPS_METRIC:
+ if (metrics.pinned_icaps_metric.updated) {
+ c->first = metrics.pinned_icaps_metric.pinned_icaps;
+ c->second = metrics.pinned_icaps_metric.total_inodes;
+ }
+ break;
+ case MDSPerformanceCounterType::OPENED_INODES_METRIC:
+ if (metrics.opened_inodes_metric.updated) {
+ c->first = metrics.opened_inodes_metric.opened_inodes;
+ c->second = metrics.opened_inodes_metric.total_inodes;
+ }
+ break;
default:
ceph_abort_msg("unknown counter type");
}
metrics.write_latency_metric = { };
metrics.metadata_latency_metric = { };
metrics.dentry_lease_metric = { };
+ metrics.opened_files_metric = { };
+ metrics.pinned_icaps_metric = { };
+ metrics.opened_inodes_metric = { };
metrics.update_type = UPDATE_TYPE_REMOVE;
}
metrics.dentry_lease_metric.updated = true;
}
+void MetricsHandler::handle_payload(Session *session, const OpenedFilesPayload &payload) {
+ dout(20) << ": type=" << static_cast<ClientMetricType>(OpenedFilesPayload::METRIC_TYPE)
+ << ", session=" << session << ", opened_files=" << payload.opened_files
+ << ", total_inodes=" << payload.total_inodes << dendl;
+
+ auto it = client_metrics_map.find(session->info.inst);
+ if (it == client_metrics_map.end()) {
+ return;
+ }
+
+ auto &metrics = it->second.second;
+ metrics.update_type = UPDATE_TYPE_REFRESH;
+ metrics.opened_files_metric.opened_files = payload.opened_files;
+ metrics.opened_files_metric.total_inodes = payload.total_inodes;
+ metrics.opened_files_metric.updated = true;
+}
+
+void MetricsHandler::handle_payload(Session *session, const PinnedIcapsPayload &payload) {
+ dout(20) << ": type=" << static_cast<ClientMetricType>(PinnedIcapsPayload::METRIC_TYPE)
+ << ", session=" << session << ", pinned_icaps=" << payload.pinned_icaps
+ << ", total_inodes=" << payload.total_inodes << dendl;
+
+ auto it = client_metrics_map.find(session->info.inst);
+ if (it == client_metrics_map.end()) {
+ return;
+ }
+
+ auto &metrics = it->second.second;
+ metrics.update_type = UPDATE_TYPE_REFRESH;
+ metrics.pinned_icaps_metric.pinned_icaps = payload.pinned_icaps;
+ metrics.pinned_icaps_metric.total_inodes = payload.total_inodes;
+ metrics.pinned_icaps_metric.updated = true;
+}
+
+void MetricsHandler::handle_payload(Session *session, const OpenedInodesPayload &payload) {
+ dout(20) << ": type=" << static_cast<ClientMetricType>(OpenedInodesPayload::METRIC_TYPE)
+ << ", session=" << session << ", opened_inodes=" << payload.opened_inodes
+ << ", total_inodes=" << payload.total_inodes << dendl;
+
+ auto it = client_metrics_map.find(session->info.inst);
+ if (it == client_metrics_map.end()) {
+ return;
+ }
+
+ auto &metrics = it->second.second;
+ metrics.update_type = UPDATE_TYPE_REFRESH;
+ metrics.opened_inodes_metric.opened_inodes = payload.opened_inodes;
+ metrics.opened_inodes_metric.total_inodes = payload.total_inodes;
+ metrics.opened_inodes_metric.updated = true;
+}
+
void MetricsHandler::handle_payload(Session *session, const UnknownPayload &payload) {
dout(5) << ": type=Unknown, session=" << session << ", ignoring unknown payload" << dendl;
}
void handle_payload(Session *session, const WriteLatencyPayload &payload);
void handle_payload(Session *session, const MetadataLatencyPayload &payload);
void handle_payload(Session *session, const DentryLeasePayload &payload);
+ void handle_payload(Session *session, const OpenedFilesPayload &payload);
+ void handle_payload(Session *session, const PinnedIcapsPayload &payload);
+ void handle_payload(Session *session, const OpenedInodesPayload &payload);
void handle_payload(Session *session, const UnknownPayload &payload);
void set_next_seq(version_t seq);
CLIENT_METRIC_TYPE_WRITE_LATENCY, \
CLIENT_METRIC_TYPE_METADATA_LATENCY, \
CLIENT_METRIC_TYPE_DENTRY_LEASE, \
+ CLIENT_METRIC_TYPE_OPENED_FILES, \
+ CLIENT_METRIC_TYPE_PINNED_ICAPS, \
+ CLIENT_METRIC_TYPE_OPENED_INODES, \
}
#define CEPHFS_FEATURES_MDS_SUPPORTED CEPHFS_FEATURES_ALL
{"write_latency", MDSPerformanceCounterType::WRITE_LATENCY_METRIC},
{"metadata_latency", MDSPerformanceCounterType::METADATA_LATENCY_METRIC},
{"dentry_lease", MDSPerformanceCounterType::DENTRY_LEASE_METRIC},
+ {"opened_files", MDSPerformanceCounterType::OPENED_FILES_METRIC},
+ {"pinned_icaps", MDSPerformanceCounterType::PINNED_ICAPS_METRIC},
+ {"opened_inodes", MDSPerformanceCounterType::OPENED_INODES_METRIC},
};
PyObject *py_query = nullptr;
case MDSPerformanceCounterType::WRITE_LATENCY_METRIC:
case MDSPerformanceCounterType::METADATA_LATENCY_METRIC:
case MDSPerformanceCounterType::DENTRY_LEASE_METRIC:
+ case MDSPerformanceCounterType::OPENED_FILES_METRIC:
+ case MDSPerformanceCounterType::PINNED_ICAPS_METRIC:
+ case MDSPerformanceCounterType::OPENED_INODES_METRIC:
break;
default:
ceph_abort_msg("unknown counter type");
case MDSPerformanceCounterType::WRITE_LATENCY_METRIC:
case MDSPerformanceCounterType::METADATA_LATENCY_METRIC:
case MDSPerformanceCounterType::DENTRY_LEASE_METRIC:
+ case MDSPerformanceCounterType::OPENED_FILES_METRIC:
+ case MDSPerformanceCounterType::PINNED_ICAPS_METRIC:
+ case MDSPerformanceCounterType::OPENED_INODES_METRIC:
break;
default:
ceph_abort_msg("unknown counter type");
case MDSPerformanceCounterType::DENTRY_LEASE_METRIC:
os << "dentry_lease_metric";
break;
+ case MDSPerformanceCounterType::OPENED_FILES_METRIC:
+ os << "opened_files_metric";
+ break;
+ case MDSPerformanceCounterType::PINNED_ICAPS_METRIC:
+ os << "pinned_icaps_metric";
+ break;
+ case MDSPerformanceCounterType::OPENED_INODES_METRIC:
+ os << "opened_inodes_metric";
+ break;
}
return os;
WRITE_LATENCY_METRIC = 2,
METADATA_LATENCY_METRIC = 3,
DENTRY_LEASE_METRIC = 4,
+ OPENED_FILES_METRIC = 5,
+ PINNED_ICAPS_METRIC = 6,
+ OPENED_INODES_METRIC = 7,
};
struct MDSPerformanceCounterDescriptor {
case MDSPerformanceCounterType::WRITE_LATENCY_METRIC:
case MDSPerformanceCounterType::METADATA_LATENCY_METRIC:
case MDSPerformanceCounterType::DENTRY_LEASE_METRIC:
+ case MDSPerformanceCounterType::OPENED_FILES_METRIC:
+ case MDSPerformanceCounterType::PINNED_ICAPS_METRIC:
+ case MDSPerformanceCounterType::OPENED_INODES_METRIC:
return true;
default:
return false;
'read_latency': 1,
'write_latency': 2,
'metadata_latency': 3,
- 'dentry_lease': 4})
+ 'dentry_lease': 4,
+ 'opened_files': 5,
+ 'pinned_icaps': 6,
+ 'opened_inodes': 7})
MDS_PERF_QUERY_COUNTERS = [] # type: List[str]
-MDS_GLOBAL_PERF_QUERY_COUNTERS = ['cap_hit', 'read_latency', 'write_latency', 'metadata_latency', 'dentry_lease'] # type: List[str]
+MDS_GLOBAL_PERF_QUERY_COUNTERS = ['cap_hit', 'read_latency', 'write_latency', 'metadata_latency', 'dentry_lease', 'opened_files', 'pinned_icaps', 'opened_inodes'] # type: List[str]
QUERY_EXPIRE_INTERVAL = timedelta(minutes=1)