From: Xiubo Li <xiubli@xxxxxxxxxx>
For both dentry lease and caps perf metric we will only count the
hit/miss info triggered from the vfs calls, for the cases like
request reply handling and perodically ceph_trim_dentries() we will
ignore them.
Currently only the debugfs is support and next will fulfill sending
the mertic data to MDS.
The output will be:
item total miss hit
-------------------------------------------------
d_lease 11 7 141
i_caps 10 13 2495064
Signed-off-by: Xiubo Li <xiubli@xxxxxxxxxx>
---
Changed in V2:
- switch to percpu counter to get a higher perf.
fs/ceph/acl.c | 2 +-
fs/ceph/caps.c | 24 ++++++++++++++++++++++-
fs/ceph/debugfs.c | 37 +++++++++++++++++++++++++++++++----
fs/ceph/dir.c | 39 ++++++++++++++++++++++++++-----------
fs/ceph/file.c | 10 ++++++++++
fs/ceph/mds_client.c | 46 ++++++++++++++++++++++++++++++++++++++++++--
fs/ceph/mds_client.h | 11 +++++++++++
fs/ceph/super.h | 6 ++++--
fs/ceph/xattr.c | 6 +++---
9 files changed, 157 insertions(+), 24 deletions(-)
diff --git a/fs/ceph/acl.c b/fs/ceph/acl.c
index aa55f412a6e3..b9411da0f6f2 100644
--- a/fs/ceph/acl.c
+++ b/fs/ceph/acl.c
@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,
struct ceph_inode_info *ci = ceph_inode(inode);
spin_lock(&ci->i_ceph_lock);
- if (__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 0))
+ if (__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 0, true))
set_cached_acl(inode, type, acl);
else
forget_cached_acl(inode, type);
diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c
index 3d56c1333777..41ee74d43a88 100644
--- a/fs/ceph/caps.c
+++ b/fs/ceph/caps.c
@@ -862,17 +862,22 @@ static void __touch_cap(struct ceph_cap *cap)
* front of their respective LRUs. (This is the preferred way for
* callers to check for caps they want.)
*/
-int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)
+int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch,
+ bool metric)
{
struct ceph_cap *cap;
struct rb_node *p;
int have = ci->i_snap_caps;
+ struct inode *inode = &ci->vfs_inode;
+ struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc;
if ((have & mask) == mask) {
dout("__ceph_caps_issued_mask ino 0x%lx snap issued %s"
" (mask %s)\n", ci->vfs_inode.i_ino,
ceph_cap_string(have),
ceph_cap_string(mask));
+ if (metric)
+ percpu_counter_inc(&mdsc->metric.i_caps_hit);
return 1;
}
@@ -887,6 +892,8 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)
ceph_cap_string(mask));
if (touch)
__touch_cap(cap);
+ if (metric)
+ percpu_counter_inc(&mdsc->metric.i_caps_hit);
return 1;
}
@@ -912,10 +919,14 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)
__touch_cap(cap);
}
}
+ if (metric)
+ percpu_counter_inc(&mdsc->metric.i_caps_hit);
return 1;
}
}
+ if (metric)
+ percpu_counter_inc(&mdsc->metric.i_caps_mis);
return 0;
}
@@ -2718,6 +2729,7 @@ static void check_max_size(struct inode *inode, loff_t endoff)
int ceph_try_get_caps(struct inode *inode, int need, int want,
bool nonblock, int *got)
{
+ struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc;
int ret;
BUG_ON(need & ~CEPH_CAP_FILE_RD);
@@ -2728,6 +2740,11 @@ int ceph_try_get_caps(struct inode *inode, int need, int want,
ret = try_get_cap_refs(inode, need, want, 0,
(nonblock ? NON_BLOCKING : 0), got);
+ if (ret == 1)
+ percpu_counter_inc(&mdsc->metric.i_caps_hit);
+ else
+ percpu_counter_inc(&mdsc->metric.i_caps_mis);
+
return ret == -EAGAIN ? 0 : ret;
}
@@ -2782,6 +2799,11 @@ int ceph_get_caps(struct file *filp, int need, int want,
continue;
}
+ if (ret == 1)
+ percpu_counter_inc(&fsc->mdsc->metric.i_caps_hit);
+ else
+ percpu_counter_inc(&fsc->mdsc->metric.i_caps_mis);
+
if ((fi->fmode & CEPH_FILE_MODE_WR) &&
fi->filp_gen != READ_ONCE(fsc->filp_gen)) {
if (ret >= 0 && _got)
diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c
index facb387c2735..e8fe2c90c731 100644
--- a/fs/ceph/debugfs.c
+++ b/fs/ceph/debugfs.c
@@ -124,6 +124,27 @@ static int mdsc_show(struct seq_file *s, void *p)
return 0;
}
+static int metric_show(struct seq_file *s, void *p)
+{
+ struct ceph_fs_client *fsc = s->private;
+ struct ceph_mds_client *mdsc = fsc->mdsc;
+
+ seq_printf(s, "item total miss hit\n");
+ seq_printf(s, "-------------------------------------------------\n");
+
+ seq_printf(s, "%-14s%-16lld%-16lld%lld\n", "d_lease",
+ atomic64_read(&mdsc->dentry_total_count),
+ percpu_counter_sum(&mdsc->metric.d_lease_mis),
+ percpu_counter_sum(&mdsc->metric.d_lease_hit));
+
+ seq_printf(s, "%-14s%-16d%-16lld%lld\n", "i_caps",
+ mdsc->caps_use_count,
+ percpu_counter_sum(&mdsc->metric.i_caps_mis),
+ percpu_counter_sum(&mdsc->metric.i_caps_hit));
+
+ return 0;
+}
+
static int caps_show_cb(struct inode *inode, struct ceph_cap *cap, void *p)
{
struct seq_file *s = p;
@@ -207,6 +228,7 @@ static int mds_sessions_show(struct seq_file *s, void *ptr)
CEPH_DEFINE_SHOW_FUNC(mdsmap_show)
CEPH_DEFINE_SHOW_FUNC(mdsc_show)
+CEPH_DEFINE_SHOW_FUNC(metric_show)
CEPH_DEFINE_SHOW_FUNC(caps_show)
CEPH_DEFINE_SHOW_FUNC(mds_sessions_show)
@@ -242,6 +264,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)
debugfs_remove(fsc->debugfs_mdsmap);
debugfs_remove(fsc->debugfs_mds_sessions);
debugfs_remove(fsc->debugfs_caps);
+ debugfs_remove(fsc->debugfs_metric);
debugfs_remove(fsc->debugfs_mdsc);
}
@@ -282,11 +305,17 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc)
fsc,
&mdsc_show_fops);
+ fsc->debugfs_metric = debugfs_create_file("metric",
+ 0400,
+ fsc->client->debugfs_dir,
+ fsc,
+ &metric_show_fops);
+
fsc->debugfs_caps = debugfs_create_file("caps",
- 0400,
- fsc->client->debugfs_dir,
- fsc,
- &caps_show_fops);
+ 0400,
+ fsc->client->debugfs_dir,
+ fsc,
+ &caps_show_fops);
}
diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c
index d17a789fd856..5c97bdbb0772 100644
--- a/fs/ceph/dir.c
+++ b/fs/ceph/dir.c
@@ -30,7 +30,7 @@
const struct dentry_operations ceph_dentry_ops;
static bool __dentry_lease_is_valid(struct ceph_dentry_info *di);
-static int __dir_lease_try_check(const struct dentry *dentry);
+static int __dir_lease_try_check(const struct dentry *dentry, bool metric);
/*
* Initialize ceph dentry state.
@@ -38,6 +38,8 @@ static int __dir_lease_try_check(const struct dentry *dentry);
static int ceph_d_init(struct dentry *dentry)
{
struct ceph_dentry_info *di;
+ struct ceph_fs_client *fsc = ceph_sb_to_client(dentry->d_sb);
+ struct ceph_mds_client *mdsc = fsc->mdsc;
di = kmem_cache_zalloc(ceph_dentry_cachep, GFP_KERNEL);
if (!di)
@@ -48,6 +50,9 @@ static int ceph_d_init(struct dentry *dentry)
di->time = jiffies;
dentry->d_fsdata = di;
INIT_LIST_HEAD(&di->lease_list);
+
+ atomic64_inc(&mdsc->dentry_total_count);
+
return 0;
}
@@ -341,7 +346,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)
!ceph_test_mount_opt(fsc, NOASYNCREADDIR) &&
ceph_snap(inode) != CEPH_SNAPDIR &&
__ceph_dir_is_complete_ordered(ci) &&
- __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) {
+ __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1, true)) {
int shared_gen = atomic_read(&ci->i_shared_gen);
spin_unlock(&ci->i_ceph_lock);
err = __dcache_readdir(file, ctx, shared_gen);
@@ -759,7 +764,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry,
!is_root_ceph_dentry(dir, dentry) &&
ceph_test_mount_opt(fsc, DCACHE) &&
__ceph_dir_is_complete(ci) &&
- (__ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {
+ (__ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1,
+ true))) {
spin_unlock(&ci->i_ceph_lock);
dout(" dir %p complete, -ENOENT\n", dir);
d_add(dentry, NULL);
@@ -1336,7 +1342,7 @@ static int __dentry_lease_check(struct dentry *dentry, void *arg)
if (__dentry_lease_is_valid(di))
return STOP;
- ret = __dir_lease_try_check(dentry);
+ ret = __dir_lease_try_check(dentry, false);
if (ret == -EBUSY)
return KEEP;
if (ret > 0)
@@ -1349,7 +1355,7 @@ static int __dir_lease_check(struct dentry *dentry, void *arg)
struct ceph_lease_walk_control *lwc = arg;
struct ceph_dentry_info *di = ceph_dentry(dentry);
- int ret = __dir_lease_try_check(dentry);
+ int ret = __dir_lease_try_check(dentry, false);
if (ret == -EBUSY)
return KEEP;
if (ret > 0) {
@@ -1488,7 +1494,7 @@ static int dentry_lease_is_valid(struct dentry *dentry, unsigned int flags)
/*
* Called under dentry->d_lock.
*/
-static int __dir_lease_try_check(const struct dentry *dentry)
+static int __dir_lease_try_check(const struct dentry *dentry, bool metric)
{
struct ceph_dentry_info *di = ceph_dentry(dentry);
struct inode *dir;
@@ -1505,7 +1511,8 @@ static int __dir_lease_try_check(const struct dentry *dentry)
if (spin_trylock(&ci->i_ceph_lock)) {
if (atomic_read(&ci->i_shared_gen) == di->lease_shared_gen &&
- __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 0))
+ __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 0,
+ metric))
valid = 1;
spin_unlock(&ci->i_ceph_lock);
} else {
@@ -1527,7 +1534,7 @@ static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry)
int shared_gen;
spin_lock(&ci->i_ceph_lock);
- valid = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1);
+ valid = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1, true);
shared_gen = atomic_read(&ci->i_shared_gen);
spin_unlock(&ci->i_ceph_lock);
if (valid) {
@@ -1551,6 +1558,7 @@ static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry)
*/
static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)
{
+ struct ceph_mds_client *mdsc;
int valid = 0;
struct dentry *parent;
struct inode *dir, *inode;
@@ -1567,6 +1575,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)
inode = d_inode(dentry);
}
+ mdsc = ceph_sb_to_client(dir->i_sb)->mdsc;
+
dout("d_revalidate %p '%pd' inode %p offset %lld\n", dentry,
dentry, inode, ceph_dentry(dentry)->offset);
@@ -1590,12 +1600,12 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)
}
if (!valid) {
- struct ceph_mds_client *mdsc =
- ceph_sb_to_client(dir->i_sb)->mdsc;
struct ceph_mds_request *req;
int op, err;
u32 mask;
+ percpu_counter_inc(&mdsc->metric.d_lease_mis);
+
if (flags & LOOKUP_RCU)
return -ECHILD;
@@ -1630,6 +1640,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)
dout("d_revalidate %p lookup result=%d\n",
dentry, err);
}
+ } else {
+ percpu_counter_inc(&mdsc->metric.d_lease_hit);
}
dout("d_revalidate %p %s\n", dentry, valid ? "valid" : "invalid");
@@ -1638,6 +1650,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)
if (!(flags & LOOKUP_RCU))
dput(parent);
+
return valid;
}
@@ -1660,7 +1673,7 @@ static int ceph_d_delete(const struct dentry *dentry)
if (di) {
if (__dentry_lease_is_valid(di))
return 0;
- if (__dir_lease_try_check(dentry))
+ if (__dir_lease_try_check(dentry, true))
return 0;
}
return 1;
@@ -1672,9 +1685,13 @@ static int ceph_d_delete(const struct dentry *dentry)
static void ceph_d_release(struct dentry *dentry)
{
struct ceph_dentry_info *di = ceph_dentry(dentry);
+ struct ceph_fs_client *fsc = ceph_sb_to_client(dentry->d_sb);
+ struct ceph_mds_client *mdsc = fsc->mdsc;
dout("d_release %p\n", dentry);
+ atomic64_dec(&mdsc->dentry_total_count);
+
spin_lock(&dentry->d_lock);
__dentry_lease_unlist(di);
dentry->d_fsdata = NULL;
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index 8de633964dc3..68cc385992d6 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -375,6 +375,9 @@ int ceph_open(struct inode *inode, struct file *file)
spin_lock(&ci->i_ceph_lock);
__ceph_get_fmode(ci, fmode);
spin_unlock(&ci->i_ceph_lock);
+
+ percpu_counter_inc(&mdsc->metric.i_caps_hit);
+
return ceph_init_file(inode, file, fmode);
}
@@ -395,6 +398,8 @@ int ceph_open(struct inode *inode, struct file *file)
__ceph_get_fmode(ci, fmode);
spin_unlock(&ci->i_ceph_lock);
+ percpu_counter_inc(&mdsc->metric.i_caps_hit);
+
/* adjust wanted? */
if ((issued & wanted) != wanted &&
(mds_wanted & wanted) != wanted &&
@@ -406,11 +411,16 @@ int ceph_open(struct inode *inode, struct file *file)
(ci->i_snap_caps & wanted) == wanted) {
__ceph_get_fmode(ci, fmode);
spin_unlock(&ci->i_ceph_lock);
+
+ percpu_counter_inc(&mdsc->metric.i_caps_hit);
+
return ceph_init_file(inode, file, fmode);
}
spin_unlock(&ci->i_ceph_lock);
+ percpu_counter_inc(&mdsc->metric.i_caps_mis);
+
dout("open fmode %d wants %s\n", fmode, ceph_cap_string(wanted));
req = prepare_open_request(inode->i_sb, flags, 0);
if (IS_ERR(req)) {
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index d8bb3eebfaeb..249f100deaae 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -4158,10 +4158,42 @@ static void delayed_work(struct work_struct *work)
schedule_delayed(mdsc);
}
+static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)
+{
+ int err;
+
+ if (!metric)
+ return -EINVAL;
+
+ err = percpu_counter_init(&metric->i_caps_hit, 0, GFP_KERNEL);
+ if (err)
+ return err;
+ err = percpu_counter_init(&metric->i_caps_mis, 0, GFP_KERNEL);
+ if (err)
+ goto err1;
+ err = percpu_counter_init(&metric->d_lease_hit, 0, GFP_KERNEL);
+ if (err)
+ goto err2;
+ err = percpu_counter_init(&metric->d_lease_mis, 0, GFP_KERNEL);
+ if (err)
+ goto err3;
+
+ return 0;
+
+err3:
+ percpu_counter_destroy(&metric->d_lease_hit);
+err2:
+ percpu_counter_destroy(&metric->i_caps_mis);
+err1:
+ percpu_counter_destroy(&metric->i_caps_hit);
+ return err;
+}
+
int ceph_mdsc_init(struct ceph_fs_client *fsc)
{
struct ceph_mds_client *mdsc;
+ int err;
mdsc = kzalloc(sizeof(struct ceph_mds_client), GFP_NOFS);
if (!mdsc)
@@ -4170,8 +4202,8 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)
mutex_init(&mdsc->mutex);
mdsc->mdsmap = kzalloc(sizeof(*mdsc->mdsmap), GFP_NOFS);
if (!mdsc->mdsmap) {
- kfree(mdsc);
- return -ENOMEM;
+ err = -ENOMEM;
+ goto err_mdsc;
}
fsc->mdsc = mdsc;
@@ -4209,6 +4241,10 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)
init_waitqueue_head(&mdsc->cap_flushing_wq);
INIT_WORK(&mdsc->cap_reclaim_work, ceph_cap_reclaim_work);
atomic_set(&mdsc->cap_reclaim_pending, 0);
+ atomic64_set(&mdsc->dentry_total_count, 0);
+ err = ceph_mdsc_metric_init(&mdsc->metric);
+ if (err)
+ goto err_mdsmap;
spin_lock_init(&mdsc->dentry_list_lock);
INIT_LIST_HEAD(&mdsc->dentry_leases);
@@ -4227,6 +4263,12 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)
strscpy(mdsc->nodename, utsname()->nodename,
sizeof(mdsc->nodename));
return 0;
+
+err_mdsmap:
+ kfree(mdsc->mdsmap);
+err_mdsc:
+ kfree(mdsc);
+ return err;
}
/*
diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h
index 9fb2063b0600..be5d3331eca3 100644
--- a/fs/ceph/mds_client.h
+++ b/fs/ceph/mds_client.h
@@ -341,6 +341,13 @@ struct ceph_quotarealm_inode {
struct inode *inode;
};
+struct ceph_client_metric {
+ struct percpu_counter d_lease_hit;
+ struct percpu_counter d_lease_mis;
+ struct percpu_counter i_caps_hit;
+ struct percpu_counter i_caps_mis;
+};
+
/*
* mds client state
*/
@@ -428,6 +435,10 @@ struct ceph_mds_client {
struct list_head dentry_leases; /* fifo list */
struct list_head dentry_dir_leases; /* lru list */
+ atomic64_t dentry_total_count;
+
+ struct ceph_client_metric metric;
+
spinlock_t snapid_map_lock;
struct rb_root snapid_map_tree;
struct list_head snapid_map_lru;
diff --git a/fs/ceph/super.h b/fs/ceph/super.h
index f0f9cb7447ac..a0e4d0bd013d 100644
--- a/fs/ceph/super.h
+++ b/fs/ceph/super.h
@@ -123,6 +123,7 @@ struct ceph_fs_client {
struct dentry *debugfs_congestion_kb;
struct dentry *debugfs_bdi;
struct dentry *debugfs_mdsc, *debugfs_mdsmap;
+ struct dentry *debugfs_metric;
struct dentry *debugfs_mds_sessions;
#endif
@@ -635,7 +636,8 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci)
}
extern int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented);
-extern int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int t);
+extern int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask,
+ int t, bool metric);
extern int __ceph_caps_issued_other(struct ceph_inode_info *ci,
struct ceph_cap *cap);
@@ -653,7 +655,7 @@ static inline int ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask,
{
int r;
spin_lock(&ci->i_ceph_lock);
- r = __ceph_caps_issued_mask(ci, mask, touch);
+ r = __ceph_caps_issued_mask(ci, mask, touch, true);
spin_unlock(&ci->i_ceph_lock);
return r;
}
diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c
index cb18ee637cb7..530fc2a72236 100644
--- a/fs/ceph/xattr.c
+++ b/fs/ceph/xattr.c
@@ -856,7 +856,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,
if (ci->i_xattrs.version == 0 ||
!((req_mask & CEPH_CAP_XATTR_SHARED) ||
- __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1))) {
+ __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1, true))) {
spin_unlock(&ci->i_ceph_lock);
/* security module gets xattr while filling trace */
@@ -914,7 +914,7 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size)
ci->i_xattrs.version, ci->i_xattrs.index_version);
if (ci->i_xattrs.version == 0 ||
- !__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1)) {
+ !__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1, true)) {
spin_unlock(&ci->i_ceph_lock);
err = ceph_do_getattr(inode, CEPH_STAT_CAP_XATTR, true);
if (err)
@@ -1192,7 +1192,7 @@ bool ceph_security_xattr_deadlock(struct inode *in)
spin_lock(&ci->i_ceph_lock);
ret = !(ci->i_ceph_flags & CEPH_I_SEC_INITED) &&
!(ci->i_xattrs.version > 0 &&
- __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 0));
+ __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 0, false));
spin_unlock(&ci->i_ceph_lock);
return ret;
}
