From: Dmitry Monakhov <dmonakhov@xxxxxxxxx>
Currently quota_hash[] is global, which is bad for scalability.
Also is is the last user of global dq_list_lock.
It is reasonable to introduce dedicated hash for each super_block
which use quota.
per-sb hash will be allocated only when necessary (on first quota_on())
Protected by per-sb dq_list_lock.
Signed-off-by: Dmitry Monakhov <dmonakhov@xxxxxxxxxx>
---
fs/quota/dquot.c | 97 ++++++++++++++++++++++++++++++-------------------
include/linux/quota.h | 11 +++++-
2 files changed, 69 insertions(+), 39 deletions(-)
diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index 324f124..822d7ad 100644
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -220,7 +220,7 @@ static void put_quota_format(struct quota_format_type *fmt)
/*
* Dquot List Management:
* The quota code uses three lists for dquot management: the inuse_list,
- * free_dquots, and dquot_hash[] array. A single dquot structure may be
+ * free_dquots, and dq_hash[] array. A single dquot structure may be
* on all three lists, depending on its current state.
*
* All dquots are placed to the end of inuse_list when first created, and this
@@ -233,13 +233,10 @@ static void put_quota_format(struct quota_format_type *fmt)
* dquot is invalidated it's completely released from memory.
*
* Dquots with a specific identity (device, type and id) are placed on
- * one of the dquot_hash[] hash chains. The provides an efficient search
+ * one of the dq_hash[] hash chains. The provides an efficient search
* mechanism to locate a specific dquot.
*/
-static unsigned int dq_hash_bits, dq_hash_mask;
-static struct hlist_head *dquot_hash;
-
struct dqstats dqstats;
EXPORT_SYMBOL(dqstats);
@@ -251,8 +248,9 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
{
unsigned long tmp;
- tmp = (((unsigned long)sb>>L1_CACHE_SHIFT) ^ id) * (MAXQUOTAS - type);
- return (tmp + (tmp >> dq_hash_bits)) & dq_hash_mask;
+ tmp = id * (MAXQUOTAS - type);
+ return (tmp + (tmp >> dqopts(sb)->dq_hash.bits)) &
+ dqopts(sb)->dq_hash.mask;
}
/*
@@ -261,7 +259,8 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
static inline void insert_dquot_hash(struct dquot *dquot)
{
struct hlist_head *head;
- head = dquot_hash + hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
+ head = sb_dqopts(dquot)->dq_hash.head +
+ hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
hlist_add_head(&dquot->dq_hash, head);
}
@@ -270,13 +269,17 @@ static inline void remove_dquot_hash(struct dquot *dquot)
hlist_del_init(&dquot->dq_hash);
}
-static struct dquot *find_dquot(unsigned int hashent, struct super_block *sb,
+static struct dquot *find_dquot(struct super_block *sb,
unsigned int id, int type)
{
struct hlist_node *node;
struct dquot *dquot;
+ unsigned int hashent = hashfn(sb, id, type);
- hlist_for_each (node, dquot_hash+hashent) {
+ if (!dqopts(sb)->dq_hash.head)
+ return NULL;
+
+ hlist_for_each(node, dqopts(sb)->dq_hash.head + hashent) {
dquot = hlist_entry(node, struct dquot, dq_hash);
if (dquot->dq_sb == sb && dquot->dq_id == id &&
dquot->dq_type == type)
@@ -854,7 +857,6 @@ static struct dquot *get_empty_dquot(struct super_block *sb, int type)
*/
struct dquot *dqget(struct super_block *sb, unsigned int id, int type)
{
- unsigned int hashent = hashfn(sb, id, type);
struct dquot *dquot = NULL, *empty = NULL;
struct quota_info *dqopt = dqopts(sb);
@@ -872,7 +874,7 @@ we_slept:
}
spin_unlock(&dqopt->dq_state_lock);
- dquot = find_dquot(hashent, sb, id, type);
+ dquot = find_dquot(sb, id, type);
if (!dquot) {
if (!empty) {
spin_unlock(&dqopt->dq_list_lock);
@@ -1927,6 +1929,42 @@ const struct dquot_operations dquot_operations = {
};
EXPORT_SYMBOL(dquot_operations);
+int alloc_quota_hash(struct quota_info *dqopt, int order)
+{
+ unsigned long nr_hash, i;
+ struct hlist_head *hash_array;
+ struct dquot_hash *dq_hash = &dqopt->dq_hash;
+
+ hash_array = (struct hlist_head *)__get_free_pages(GFP_KERNEL, order);
+ if (!hash_array)
+ return -ENOMEM;
+
+ /* Find power-of-two hlist_heads which can fit into allocation */
+ nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
+ dq_hash->bits = ilog2(nr_hash);
+ nr_hash = 1UL << dq_hash->bits;
+ dq_hash->mask = nr_hash - 1;
+ for (i = 0; i < nr_hash; i++)
+ INIT_HLIST_HEAD(hash_array + i);
+ dq_hash->order = order;
+ dq_hash->head = hash_array;
+ return 0;
+}
+
+void free_quota_hash(struct quota_info *dqopt)
+{
+
+ struct dquot_hash *dq_hash = &dqopt->dq_hash;
+ unsigned long i, nr_hash = 1UL << dq_hash->bits;
+ unsigned long addr = (unsigned long )dq_hash->head;
+
+ for (i = 0; i < nr_hash; i++)
+ WARN_ON(!hlist_empty(dq_hash->head + i));
+
+ dq_hash->head = NULL;
+ free_pages(addr, dq_hash->order);
+}
+
/*
* Generic helper for ->open on filesystems supporting disk quotas.
*/
@@ -1956,14 +1994,21 @@ static int alloc_quota_info(struct quota_ctl_info *dqctl) {
spin_lock_init(&dqopt->dq_list_lock);
INIT_LIST_HEAD(&dqopt->dq_inuse_list);
INIT_LIST_HEAD(&dqopt->dq_free_list);
-
dqctl->dq_opt = dqopt;
- return 0;
+ err = alloc_quota_hash(dqopt, 0);
+
+ if (err && dqopt) {
+ free_quota_hash(dqopt);
+ kfree(dqopt);
+ dqctl->dq_opt = NULL;
+ }
+ return err;
}
static void free_quota_info(struct quota_ctl_info *dqctl)
{
if (dqctl->dq_opt) {
+ free_quota_hash(dqctl->dq_opt);
kfree(dqctl->dq_opt);
dqctl->dq_opt = NULL;
}
@@ -2709,8 +2754,6 @@ static ctl_table sys_table[] = {
static int __init dquot_init(void)
{
int i, ret;
- unsigned long nr_hash, order;
-
printk(KERN_NOTICE "VFS: Disk quotas %s\n", __DQUOT_VERSION__);
register_sysctl_table(sys_table);
@@ -2721,33 +2764,11 @@ static int __init dquot_init(void)
SLAB_MEM_SPREAD|SLAB_PANIC),
NULL);
- order = 0;
- dquot_hash = (struct hlist_head *)__get_free_pages(GFP_ATOMIC, order);
- if (!dquot_hash)
- panic("Cannot create dquot hash table");
-
for (i = 0; i < _DQST_DQSTAT_LAST; i++) {
ret = percpu_counter_init(&dqstats.counter[i], 0);
if (ret)
panic("Cannot create dquot stat counters");
}
-
- /* Find power-of-two hlist_heads which can fit into allocation */
- nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
- dq_hash_bits = 0;
- do {
- dq_hash_bits++;
- } while (nr_hash >> dq_hash_bits);
- dq_hash_bits--;
-
- nr_hash = 1UL << dq_hash_bits;
- dq_hash_mask = nr_hash - 1;
- for (i = 0; i < nr_hash; i++)
- INIT_HLIST_HEAD(dquot_hash + i);
-
- printk("Dquot-cache hash table entries: %ld (order %ld, %ld bytes)\n",
- nr_hash, order, (PAGE_SIZE << order));
-
register_shrinker(&dqcache_shrinker);
return 0;
diff --git a/include/linux/quota.h b/include/linux/quota.h
index bb63abf..eaa9f91 100644
--- a/include/linux/quota.h
+++ b/include/linux/quota.h
@@ -401,13 +401,22 @@ struct quota_ctl_info {
const struct dquot_operations *dq_op;
struct quota_info *dq_opt;
};
+
+struct dquot_hash {
+ struct hlist_head *head;
+ unsigned int order;
+ unsigned int bits;
+ unsigned int mask;
+};
+
struct quota_info {
struct mutex dqio_mutex; /* lock device while I/O in progress */
struct mem_dqinfo info[MAXQUOTAS]; /* Information for each quota type */
spinlock_t dq_state_lock; /* serialize quota state changes*/
- spinlock_t dq_list_lock; /* protect lists */
+ spinlock_t dq_list_lock; /* protect lists and hash*/
struct list_head dq_inuse_list; /* list of inused dquotas */
struct list_head dq_free_list; /* list of free dquotas */
+ struct dquot_hash dq_hash; /* dquot lookup hash */
struct inode *files[MAXQUOTAS]; /* inodes of quotafiles */
const struct quota_format_ops *fmt_ops[MAXQUOTAS]; /* Operations for each type */
--
1.6.5.2
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
