Currently quota_hash[] is global, which is bad for scalability.
Also is is the last user of global dq_list_lock.
It is reasonable to introduce didicated hash for each super_block
which use quota.
per-sb hash will be allocated only when necessery (on first quota_on())
Protected by per-sb dq_list_lock.
Signed-off-by: Dmitry Monakhov <dmonakhov@xxxxxxxxx>
---
fs/quota/dquot.c | 111 ++++++++++++++++++++++++++++++++----------------
include/linux/quota.h | 10 ++++-
2 files changed, 83 insertions(+), 38 deletions(-)
diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index f2092d1..e93b323 100644
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -220,7 +220,7 @@ static void put_quota_format(struct quota_format_type *fmt)
/*
* Dquot List Management:
* The quota code uses three lists for dquot management: the inuse_list,
- * free_dquots, and dquot_hash[] array. A single dquot structure may be
+ * free_dquots, and dq_hash[] array. A single dquot structure may be
* on all three lists, depending on its current state.
*
* All dquots are placed to the end of inuse_list when first created, and this
@@ -233,13 +233,10 @@ static void put_quota_format(struct quota_format_type *fmt)
* dquot is invalidated it's completely released from memory.
*
* Dquots with a specific identity (device, type and id) are placed on
- * one of the dquot_hash[] hash chains. The provides an efficient search
+ * one of the dq_hash[] hash chains. The provides an efficient search
* mechanism to locate a specific dquot.
*/
-static unsigned int dq_hash_bits, dq_hash_mask;
-static struct hlist_head *dquot_hash;
-
struct dqstats dqstats;
EXPORT_SYMBOL(dqstats);
@@ -251,8 +248,9 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
{
unsigned long tmp;
- tmp = (((unsigned long)sb>>L1_CACHE_SHIFT) ^ id) * (MAXQUOTAS - type);
- return (tmp + (tmp >> dq_hash_bits)) & dq_hash_mask;
+ tmp = id * (MAXQUOTAS - type);
+ return (tmp + (tmp >> sb->s_dquot.dq_hash.bits)) &
+ sb->s_dquot.dq_hash.mask;
}
/*
@@ -261,7 +259,8 @@ hashfn(const struct super_block *sb, unsigned int id, int type)
static inline void insert_dquot_hash(struct dquot *dquot)
{
struct hlist_head *head;
- head = dquot_hash + hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
+ head = dq_opt(dquot)->dq_hash.head +
+ hashfn(dquot->dq_sb, dquot->dq_id, dquot->dq_type);
hlist_add_head(&dquot->dq_hash, head);
}
@@ -270,13 +269,17 @@ static inline void remove_dquot_hash(struct dquot *dquot)
hlist_del_init(&dquot->dq_hash);
}
-static struct dquot *find_dquot(unsigned int hashent, struct super_block *sb,
+static struct dquot *find_dquot(struct super_block *sb,
unsigned int id, int type)
{
struct hlist_node *node;
struct dquot *dquot;
+ unsigned int hashent = hashfn(sb, id, type);
+
+ if (!sb_dqopt(sb)->dq_hash.head)
+ return NULL;
- hlist_for_each (node, dquot_hash+hashent) {
+ hlist_for_each(node, sb_dqopt(sb)->dq_hash.head + hashent) {
dquot = hlist_entry(node, struct dquot, dq_hash);
if (dquot->dq_sb == sb && dquot->dq_id == id &&
dquot->dq_type == type)
@@ -846,7 +849,6 @@ static struct dquot *get_empty_dquot(struct super_block *sb, int type)
*/
struct dquot *dqget(struct super_block *sb, unsigned int id, int type)
{
- unsigned int hashent = hashfn(sb, id, type);
struct dquot *dquot = NULL, *empty = NULL;
struct quota_info *dqopt = sb_dqopt(sb);
@@ -864,7 +866,7 @@ we_slept:
}
spin_unlock(&sb_dqopt(sb)->dq_state_lock);
- dquot = find_dquot(hashent, sb, id, type);
+ dquot = find_dquot(sb, id, type);
if (!dquot) {
if (!empty) {
spin_unlock(&dqopt->dq_list_lock);
@@ -1925,6 +1927,47 @@ int dquot_file_open(struct inode *inode, struct file *file)
}
EXPORT_SYMBOL(dquot_file_open);
+int dquot_hash_alloc(struct super_block *sb, int order)
+{
+ unsigned long nr_hash, i;
+ struct hlist_head *hash_array;
+ struct dquot_hash *dq_hash = &sb_dqopt(sb)->dq_hash;
+
+ hash_array = (struct hlist_head *)__get_free_pages(GFP_KERNEL, order);
+ if (!hash_array)
+ return -ENOMEM;
+
+ /* Find power-of-two hlist_heads which can fit into allocation */
+ nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
+ dq_hash->bits = 0;
+ do {
+ dq_hash->bits++;
+ } while (nr_hash >> dq_hash->bits);
+ dq_hash->bits--;
+
+ nr_hash = 1UL << dq_hash->bits;
+ dq_hash->mask = nr_hash - 1;
+ for (i = 0; i < nr_hash; i++)
+ INIT_HLIST_HEAD(hash_array + i);
+ dq_hash->order = order;
+ dq_hash->head = hash_array;
+ return 0;
+}
+
+void dquot_hash_destroy(struct super_block *sb)
+{
+
+ struct dquot_hash * dq_hash = &sb_dqopt(sb)->dq_hash;
+ unsigned long i, nr_hash = 1UL << dq_hash->bits;
+ unsigned long addr = (unsigned long )dq_hash->head;
+
+ for (i = 0; i < nr_hash; i++)
+ BUG_ON(!hlist_empty(dq_hash->head + i));
+
+ dq_hash->head = NULL;
+ free_pages(addr, dq_hash->order);
+}
+
/*
* Turn quota off on a device. type == -1 ==> quotaoff for all types (umount)
*/
@@ -2006,6 +2049,9 @@ int dquot_disable(struct super_block *sb, int type, unsigned int flags)
dqopt->info[cnt].dqi_bgrace = 0;
dqopt->ops[cnt] = NULL;
}
+ if (!sb_any_quota_loaded(sb))
+ /* All quotas was unloaded, hash no longer needed */
+ dquot_hash_destroy(sb);
mutex_unlock(&dqopt->dqonoff_mutex);
/* Skip syncing and setting flags if quota files are hidden */
@@ -2081,7 +2127,7 @@ static int vfs_load_quota_inode(struct inode *inode, int type, int format_id,
struct quota_format_type *fmt = find_quota_format(format_id);
struct super_block *sb = inode->i_sb;
struct quota_info *dqopt = sb_dqopt(sb);
- int error;
+ int error, hash_alloc;
int oldflags = -1;
if (!fmt)
@@ -2115,6 +2161,16 @@ static int vfs_load_quota_inode(struct inode *inode, int type, int format_id,
invalidate_bdev(sb->s_bdev);
}
mutex_lock(&dqopt->dqonoff_mutex);
+
+ hash_alloc = !sb_any_quota_loaded(sb);
+ if (hash_alloc) {
+ error = dquot_hash_alloc(sb, 0);
+ if (error) {
+ hash_alloc = 0;
+ goto out_lock;
+ }
+ }
+
if (sb_has_quota_loaded(sb, type)) {
error = -EBUSY;
goto out_lock;
@@ -2168,6 +2224,8 @@ out_file_init:
dqopt->files[type] = NULL;
iput(inode);
out_lock:
+ if (hash_alloc)
+ dquot_hash_destroy(sb);
if (oldflags != -1) {
mutex_lock_nested(&inode->i_mutex, I_MUTEX_QUOTA);
/* Set the flags back (in the case of accidental quotaon()
@@ -2224,8 +2282,10 @@ int dquot_quota_on(struct super_block *sb, int type, int format_id,
struct path *path)
{
int error = security_quota_on(path->dentry);
+
if (error)
return error;
+
/* Quota file not on the same filesystem? */
if (path->mnt->mnt_sb != sb)
error = -EXDEV;
@@ -2643,8 +2703,7 @@ static ctl_table sys_table[] = {
static int __init dquot_init(void)
{
int i, ret;
- unsigned long nr_hash, order;
-
+
printk(KERN_NOTICE "VFS: Disk quotas %s\n", __DQUOT_VERSION__);
register_sysctl_table(sys_table);
@@ -2655,33 +2714,11 @@ static int __init dquot_init(void)
SLAB_MEM_SPREAD|SLAB_PANIC),
NULL);
- order = 0;
- dquot_hash = (struct hlist_head *)__get_free_pages(GFP_ATOMIC, order);
- if (!dquot_hash)
- panic("Cannot create dquot hash table");
-
for (i = 0; i < _DQST_DQSTAT_LAST; i++) {
ret = percpu_counter_init(&dqstats.counter[i], 0);
if (ret)
panic("Cannot create dquot stat counters");
}
-
- /* Find power-of-two hlist_heads which can fit into allocation */
- nr_hash = (1UL << order) * PAGE_SIZE / sizeof(struct hlist_head);
- dq_hash_bits = 0;
- do {
- dq_hash_bits++;
- } while (nr_hash >> dq_hash_bits);
- dq_hash_bits--;
-
- nr_hash = 1UL << dq_hash_bits;
- dq_hash_mask = nr_hash - 1;
- for (i = 0; i < nr_hash; i++)
- INIT_HLIST_HEAD(dquot_hash + i);
-
- printk("Dquot-cache hash table entries: %ld (order %ld, %ld bytes)\n",
- nr_hash, order, (PAGE_SIZE << order));
-
register_shrinker(&dqcache_shrinker);
return 0;
diff --git a/include/linux/quota.h b/include/linux/quota.h
index 134c18d..eb083fc 100644
--- a/include/linux/quota.h
+++ b/include/linux/quota.h
@@ -393,15 +393,23 @@ static inline void quota_send_warning(short type, unsigned int id, dev_t dev,
}
#endif /* CONFIG_QUOTA_NETLINK_INTERFACE */
+struct dquot_hash {
+ struct hlist_head *head;
+ unsigned int order;
+ unsigned int bits;
+ unsigned int mask;
+};
+
struct quota_info {
unsigned int flags; /* Flags for diskquotas on this device */
struct mutex dqio_mutex; /* lock device while I/O in progress */
struct mutex dqonoff_mutex; /* Serialize quotaon & quotaoff */
struct rw_semaphore dqptr_sem; /* serialize ops using quota_info struct, pointers from inode to dquots */
spinlock_t dq_state_lock; /* serialize quota state changes*/
- spinlock_t dq_list_lock; /* protect lists */
+ spinlock_t dq_list_lock; /* protect lists and hash */
struct list_head dq_inuse_list; /* list of inused dquotas */
struct list_head dq_free_list; /* list of free dquotas */
+ struct dquot_hash dq_hash; /* dquot lookup hash */
struct inode *files[MAXQUOTAS]; /* inodes of quotafiles */
struct mem_dqinfo info[MAXQUOTAS]; /* Information for each quota type */
--
1.6.6.1
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
