diff options
author | Ingo Molnar <mingo@elte.hu> | 2006-07-10 04:44:04 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@g5.osdl.org> | 2006-07-10 13:24:14 -0700 |
commit | d6d897cec29252b8d0785198cfa6ca16d30c739d (patch) | |
tree | 01b138b10c7afa0e88e9720be3d3616e4f0bccc1 | |
parent | 55794a412fdf9af1744800e5020a4ec6b21e3cdc (diff) | |
download | linux-d6d897cec29252b8d0785198cfa6ca16d30c739d.tar.bz2 |
[PATCH] lockdep: core, reduce per-lock class-cache size
lockdep_map is embedded into every lock, which blows up data structure
sizes all around the kernel. Reduce the class-cache to be for the default
class only - that is used in 99.9% of the cases and even if we dont have a
class cached, the lookup in the class-hash is lockless.
This change reduces the per-lock dep_map overhead by 56 bytes on 64-bit
platforms and by 28 bytes on 32-bit platforms.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r-- | include/linux/lockdep.h | 2 | ||||
-rw-r--r-- | kernel/lockdep.c | 87 |
2 files changed, 55 insertions, 34 deletions
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 316e0fb8d7b1..c040a8c969aa 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -120,7 +120,7 @@ struct lock_class { */ struct lockdep_map { struct lock_class_key *key; - struct lock_class *class[MAX_LOCKDEP_SUBCLASSES]; + struct lock_class *class_cache; const char *name; }; diff --git a/kernel/lockdep.c b/kernel/lockdep.c index c1f34addd003..9bad17884513 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -1104,7 +1104,7 @@ extern void __error_too_big_MAX_LOCKDEP_SUBCLASSES(void); * itself, so actual lookup of the hash should be once per lock object. */ static inline struct lock_class * -register_lock_class(struct lockdep_map *lock, unsigned int subclass) +look_up_lock_class(struct lockdep_map *lock, unsigned int subclass) { struct lockdep_subclass_key *key; struct list_head *hash_head; @@ -1148,7 +1148,26 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass) */ list_for_each_entry(class, hash_head, hash_entry) if (class->key == key) - goto out_set; + return class; + + return NULL; +} + +/* + * Register a lock's class in the hash-table, if the class is not present + * yet. Otherwise we look it up. We cache the result in the lock object + * itself, so actual lookup of the hash should be once per lock object. + */ +static inline struct lock_class * +register_lock_class(struct lockdep_map *lock, unsigned int subclass) +{ + struct lockdep_subclass_key *key; + struct list_head *hash_head; + struct lock_class *class; + + class = look_up_lock_class(lock, subclass); + if (likely(class)) + return class; /* * Debug-check: all keys must be persistent! @@ -1163,6 +1182,9 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass) return NULL; } + key = lock->key->subkeys + subclass; + hash_head = classhashentry(key); + __raw_spin_lock(&hash_lock); /* * We have to do the hash-walk again, to avoid races @@ -1209,8 +1231,8 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass) out_unlock_set: __raw_spin_unlock(&hash_lock); -out_set: - lock->class[subclass] = class; + if (!subclass) + lock->class_cache = class; DEBUG_LOCKS_WARN_ON(class->subclass != subclass); @@ -1914,7 +1936,7 @@ void lockdep_init_map(struct lockdep_map *lock, const char *name, } lock->name = name; lock->key = key; - memset(lock->class, 0, sizeof(lock->class[0])*MAX_LOCKDEP_SUBCLASSES); + lock->class_cache = NULL; } EXPORT_SYMBOL_GPL(lockdep_init_map); @@ -1928,8 +1950,8 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, unsigned long ip) { struct task_struct *curr = current; + struct lock_class *class = NULL; struct held_lock *hlock; - struct lock_class *class; unsigned int depth, id; int chain_head = 0; u64 chain_key; @@ -1947,8 +1969,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, return 0; } - class = lock->class[subclass]; - /* not cached yet? */ + if (!subclass) + class = lock->class_cache; + /* + * Not cached yet or subclass? + */ if (unlikely(!class)) { class = register_lock_class(lock, subclass); if (!class) @@ -2449,48 +2474,44 @@ void lockdep_free_key_range(void *start, unsigned long size) void lockdep_reset_lock(struct lockdep_map *lock) { - struct lock_class *class, *next, *entry; + struct lock_class *class, *next; struct list_head *head; unsigned long flags; int i, j; raw_local_irq_save(flags); - __raw_spin_lock(&hash_lock); /* - * Remove all classes this lock has: + * Remove all classes this lock might have: + */ + for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) { + /* + * If the class exists we look it up and zap it: + */ + class = look_up_lock_class(lock, j); + if (class) + zap_class(class); + } + /* + * Debug check: in the end all mapped classes should + * be gone. */ + __raw_spin_lock(&hash_lock); for (i = 0; i < CLASSHASH_SIZE; i++) { head = classhash_table + i; if (list_empty(head)) continue; list_for_each_entry_safe(class, next, head, hash_entry) { - for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) { - entry = lock->class[j]; - if (class == entry) { - zap_class(class); - lock->class[j] = NULL; - break; - } + if (unlikely(class == lock->class_cache)) { + __raw_spin_unlock(&hash_lock); + DEBUG_LOCKS_WARN_ON(1); + goto out_restore; } } } - - /* - * Debug check: in the end all mapped classes should - * be gone. - */ - for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) { - entry = lock->class[j]; - if (!entry) - continue; - __raw_spin_unlock(&hash_lock); - DEBUG_LOCKS_WARN_ON(1); - raw_local_irq_restore(flags); - return; - } - __raw_spin_unlock(&hash_lock); + +out_restore: raw_local_irq_restore(flags); } |