The patch titled
lockdep: core, reduce per-lock class-cache size
has been added to the -mm tree. Its filename is
lockdep-core-reduce-per-lock-class-cache-size.patch
See http://www.zip.com.au/~akpm/linux/patches/stuff/added-to-mm.txt to find
out what to do about this
------------------------------------------------------
Subject: lockdep: core, reduce per-lock class-cache size
From: Ingo Molnar <mingo@xxxxxxx>
lockdep_map is embedded into every lock, which blows up data structure
sizes all around the kernel. Reduce the class-cache to be for the default
class only - that is used in 99.9% of the cases and even if we dont have a
class cached, the lookup in the class-hash is lockless.
This change reduces the per-lock dep_map overhead by 56 bytes on 64-bit
platforms and by 28 bytes on 32-bit platforms.
Signed-off-by: Ingo Molnar <mingo@xxxxxxx>
Cc: Arjan van de Ven <arjan@xxxxxxxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxx>
---
include/linux/lockdep.h | 2
kernel/lockdep.c | 87 +++++++++++++++++++++++---------------
2 files changed, 55 insertions(+), 34 deletions(-)
diff -puN include/linux/lockdep.h~lockdep-core-reduce-per-lock-class-cache-size include/linux/lockdep.h
--- a/include/linux/lockdep.h~lockdep-core-reduce-per-lock-class-cache-size
+++ a/include/linux/lockdep.h
@@ -120,7 +120,7 @@ struct lock_class {
*/
struct lockdep_map {
struct lock_class_key *key;
- struct lock_class *class[MAX_LOCKDEP_SUBCLASSES];
+ struct lock_class *class_cache;
const char *name;
};
diff -puN kernel/lockdep.c~lockdep-core-reduce-per-lock-class-cache-size kernel/lockdep.c
--- a/kernel/lockdep.c~lockdep-core-reduce-per-lock-class-cache-size
+++ a/kernel/lockdep.c
@@ -1104,7 +1104,7 @@ extern void __error_too_big_MAX_LOCKDEP_
* itself, so actual lookup of the hash should be once per lock object.
*/
static inline struct lock_class *
-register_lock_class(struct lockdep_map *lock, unsigned int subclass)
+look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)
{
struct lockdep_subclass_key *key;
struct list_head *hash_head;
@@ -1148,7 +1148,26 @@ register_lock_class(struct lockdep_map *
*/
list_for_each_entry(class, hash_head, hash_entry)
if (class->key == key)
- goto out_set;
+ return class;
+
+ return NULL;
+}
+
+/*
+ * Register a lock's class in the hash-table, if the class is not present
+ * yet. Otherwise we look it up. We cache the result in the lock object
+ * itself, so actual lookup of the hash should be once per lock object.
+ */
+static inline struct lock_class *
+register_lock_class(struct lockdep_map *lock, unsigned int subclass)
+{
+ struct lockdep_subclass_key *key;
+ struct list_head *hash_head;
+ struct lock_class *class;
+
+ class = look_up_lock_class(lock, subclass);
+ if (likely(class))
+ return class;
/*
* Debug-check: all keys must be persistent!
@@ -1163,6 +1182,9 @@ register_lock_class(struct lockdep_map *
return NULL;
}
+ key = lock->key->subkeys + subclass;
+ hash_head = classhashentry(key);
+
__raw_spin_lock(&hash_lock);
/*
* We have to do the hash-walk again, to avoid races
@@ -1209,8 +1231,8 @@ register_lock_class(struct lockdep_map *
out_unlock_set:
__raw_spin_unlock(&hash_lock);
-out_set:
- lock->class[subclass] = class;
+ if (!subclass)
+ lock->class_cache = class;
DEBUG_LOCKS_WARN_ON(class->subclass != subclass);
@@ -1914,7 +1936,7 @@ void lockdep_init_map(struct lockdep_map
}
lock->name = name;
lock->key = key;
- memset(lock->class, 0, sizeof(lock->class[0])*MAX_LOCKDEP_SUBCLASSES);
+ lock->class_cache = NULL;
}
EXPORT_SYMBOL_GPL(lockdep_init_map);
@@ -1928,8 +1950,8 @@ static int __lock_acquire(struct lockdep
unsigned long ip)
{
struct task_struct *curr = current;
+ struct lock_class *class = NULL;
struct held_lock *hlock;
- struct lock_class *class;
unsigned int depth, id;
int chain_head = 0;
u64 chain_key;
@@ -1947,8 +1969,11 @@ static int __lock_acquire(struct lockdep
return 0;
}
- class = lock->class[subclass];
- /* not cached yet? */
+ if (!subclass)
+ class = lock->class_cache;
+ /*
+ * Not cached yet or subclass?
+ */
if (unlikely(!class)) {
class = register_lock_class(lock, subclass);
if (!class)
@@ -2449,48 +2474,44 @@ void lockdep_free_key_range(void *start,
void lockdep_reset_lock(struct lockdep_map *lock)
{
- struct lock_class *class, *next, *entry;
+ struct lock_class *class, *next;
struct list_head *head;
unsigned long flags;
int i, j;
raw_local_irq_save(flags);
- __raw_spin_lock(&hash_lock);
/*
- * Remove all classes this lock has:
+ * Remove all classes this lock might have:
*/
+ for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
+ /*
+ * If the class exists we look it up and zap it:
+ */
+ class = look_up_lock_class(lock, j);
+ if (class)
+ zap_class(class);
+ }
+ /*
+ * Debug check: in the end all mapped classes should
+ * be gone.
+ */
+ __raw_spin_lock(&hash_lock);
for (i = 0; i < CLASSHASH_SIZE; i++) {
head = classhash_table + i;
if (list_empty(head))
continue;
list_for_each_entry_safe(class, next, head, hash_entry) {
- for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
- entry = lock->class[j];
- if (class == entry) {
- zap_class(class);
- lock->class[j] = NULL;
- break;
- }
+ if (unlikely(class == lock->class_cache)) {
+ __raw_spin_unlock(&hash_lock);
+ DEBUG_LOCKS_WARN_ON(1);
+ goto out_restore;
}
}
}
-
- /*
- * Debug check: in the end all mapped classes should
- * be gone.
- */
- for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
- entry = lock->class[j];
- if (!entry)
- continue;
- __raw_spin_unlock(&hash_lock);
- DEBUG_LOCKS_WARN_ON(1);
- raw_local_irq_restore(flags);
- return;
- }
-
__raw_spin_unlock(&hash_lock);
+
+out_restore:
raw_local_irq_restore(flags);
}
_
Patches currently in -mm which might be from mingo@xxxxxxx are
origin.patch
disable-debugging-version-of-write_lock.patch
my-name-is-ingo-molnar-you-killed-my-make-allyesconfig-prepare-to-die.patch
lock-validator-fix-ns83820c-irq-flags-bug.patch
lockdep-fix-atm-ipcommonc-deadlock.patch
lockdep-annotate-8390c-disable_irq-2.patch
lockdep-fix-8139tooc-deadlock.patch
lockdep-fix-sk_dst_check-deadlock.patch
revert-gregkh-pci-pci-test-that-drivers-properly-call-pci_set_master.patch
sched-fix-bug-in-__migrate_task.patch
small-kernel-schedc-cleanup.patch
x86-re-enable-generic-numa.patch
minor-cleanup-to-lockdepc.patch
lockdep-add-more-rwsemh-documentation.patch
improve-lockdep-debug-output.patch
lockdep-core-reduce-per-lock-class-cache-size.patch
sched-add-above-background-load-function.patch
mm-implement-swap-prefetching.patch
sched-cleanup-remove-task_t-convert-to-struct-task_struct-prefetch.patch
genirq-convert-the-x86_64-architecture-to-irq-chips.patch
genirq-convert-the-i386-architecture-to-irq-chips.patch
genirq-irq-convert-the-move_irq-flag-from-a-32bit-word-to-a-single-bit.patch
genirq-irq-add-moved_masked_irq.patch
genirq-x86_64-irq-reenable-migrating-irqs-to-other-cpus.patch
genirq-msi-simplify-msi-enable-and-disable.patch
genirq-msi-make-the-msi-boolean-tests-return-either-0-or-1.patch
genirq-msi-implement-helper-functions-read_msi_msg-and-write_msi_msg.patch
genirq-msi-refactor-the-msi_ops.patch
genirq-msi-simplify-the-msi-irq-limit-policy.patch
genirq-irq-add-a-dynamic-irq-creation-api.patch
genirq-ia64-irq-dynamic-irq-support.patch
genirq-i386-irq-dynamic-irq-support.patch
genirq-x86_64-irq-dynamic-irq-support.patch
genirq-msi-make-the-msi-code-irq-based-and-not-vector-based.patch
genirq-x86_64-irq-move-msi-message-composition-into-io_apicc.patch
genirq-i386-irq-move-msi-message-composition-into-io_apicc.patch
genirq-msi-only-build-msi-apicc-on-ia64.patch
genirq-x86_64-irq-remove-the-msi-assumption-that-irq-==-vector.patch
genirq-i386-irq-remove-the-msi-assumption-that-irq-==-vector.patch
genirq-irq-remove-msi-hacks.patch
genirq-irq-generalize-the-check-for-hardirq_bits.patch
genirq-x86_64-irq-make-the-external-irq-handlers-report-their-vector-not-the-irq-number.patch
genirq-x86_64-irq-make-vector_irq-per-cpu.patch
genirq-x86_64-irq-kill-gsi_irq_sharing.patch
genirq-x86_64-irq-kill-irq-compression.patch
detect-atomic-counter-underflows.patch
debug-shared-irqs.patch
make-frame_pointer-default=y.patch
mutex-subsystem-synchro-test-module.patch
vdso-print-fatal-signals.patch
vdso-improve-print_fatal_signals-support-by-adding-memory-maps.patch
-
To unsubscribe from this list: send the line "unsubscribe mm-commits" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
