# # # add_file "include/linux/lock_stat.h" # content [561b86b61ac70c1ba55ff144f3ecac779934d09f] # # add_file "kernel/lock_stat.c" # content [6e4552eba62120704bbe9e785433d3716dd7b0c2] # # patch "arch/xtensa/platform-iss/network.c" # from [8c372aaff21fb65a0a03f62d19d53c1d0b1cefb8] # to [a60ede200393383d23f4d4e5ade70a6b94f580ca] # # patch "fs/dcache.c" # from [ee722526acdb0b5e749dbe54d836216e5e0c696a] # to [b01158646d18e7a951c083cfad774f5628fbcfdf] # # patch "include/linux/eventpoll.h" # from [bd142a622609d04952fac6215586fff353dab729] # to [43271ded1a3b9f40beb37aaff9e02fadeecb4655] # # patch "include/linux/mutex.h" # from [8d322d0e6ab629c98d1222fedf1d6e8ab0afbc31] # to [1d77e157da8b1716940135dfe86ce97ea2fd8d17] # # patch "include/linux/rt_lock.h" # from [e257892e9eb655e7bd12ed00ff23f5167fb97b5d] # to [420425e6ce33342d9f5eb6b2e4be80453b7139cc] # # patch "include/linux/rtmutex.h" # from [e6fa10297e6c20d27edba172aeb078a60c64488e] # to [55cd2de44a52e049fa8a0da63bde6449cefeb8fe] # # patch "include/linux/spinlock.h" # from [b4a4e821bac27625019dd765523d82f6e131f4b0] # to [144f383da63fc13b331bb5459fb66528c20e569c] # # patch "include/linux/wait.h" # from [12da8de69f1f2660443a04c3df199e5d851ea2ca] # to [9b7448af82583bd11d18032aedfa8f2af44345f4] # # patch "init/main.c" # from [aac66f499d9b75035c984a599d00646fe9b81114] # to [b0c6e65355a4c13f28fa91b9449795045aa21e93] # # patch "kernel/Kconfig.preempt" # from [3148bd94270ea0a853d8e443616cd7a668dd0d3b] # to [d63831dbfbb9e68386bfc862fd2dd1a8f1e9779f] # # patch "kernel/Makefile" # from [0690fbe8c605a1c7e24b7b94d05a96ea32574aab] # to [08087775b67b7ac1682dac0310003ef7ecbd7e70] # # patch "kernel/rt.c" # from [e7b738b39985ab0afd49068c3c937dece5249f50] # to [03449e79eedce653a4acc874f65301b17f10b2b4] # # patch "kernel/rtmutex.c" # from [5fbb6943266e0a2de638851c887e331999eaa16d] # to [bc9b7e9163572d0fb80256d6095f443788683a25] # # patch "kernel/sched.c" # from [cf65e71d0f9406fd8b7c0d142504acbe58858f93] # to [418dd6273f492b996d5665c78e48dcb88f1a9b4a] # # patch "net/tipc/node.c" # from [f60bf9ad7d6c43937acab556ff324cbb10ea46fb] # to [9944ff3bd01fba317ee36a9cba2365e3b2ef3a66] # ============================================================ --- include/linux/lock_stat.h 561b86b61ac70c1ba55ff144f3ecac779934d09f +++ include/linux/lock_stat.h 561b86b61ac70c1ba55ff144f3ecac779934d09f @@ -0,0 +1,159 @@ +/* + * By Bill Huey (hui) at + * + * Release under the what ever the Linux kernel chooses for a + * license, GNU Public License GPL v2 + * + * Tue Sep 5 17:27:48 PDT 2006 + * Created lock_stat.h + * + * Wed Sep 6 15:36:14 PDT 2006 + * Thinking about the object lifetime of a spinlock. Please refer to + * comments in kernel/lock_stat.c instead. + * + */ + +#ifndef LOCK_STAT_H +#define LOCK_STAT_H + +#ifdef CONFIG_LOCK_STAT + +#include +#include +#include +#include + +#include + +typedef struct lock_stat { + char function[KSYM_NAME_LEN]; + int line; + char *file; + + atomic_t ncontended; + atomic_t nhandoff; + unsigned int ntracked; + atomic_t ninlined; + atomic_t nspinnable; + + struct rb_node rb_node; + struct list_head list_head; +} lock_stat_t; + +enum { + LOCK_STAT_NOTE_TYPE_CONTENTION, + LOCK_STAT_NOTE_TYPE_HANDOFF +}; + +typedef lock_stat_t *lock_stat_ref_t; + +struct task_struct; + +#define LOCK_STAT_INIT(field) +#define LOCK_STAT_INITIALIZER(field) { \ + __FILE__, __FUNCTION__, __LINE__, \ + ATOMIC_INIT(0), LIST_HEAD_INIT(field) } + +#define LOCK_STAT_NOTE __FILE__, __FUNCTION__, __LINE__ +#define LOCK_STAT_NOTE_VARS _file, _function, _line +#define LOCK_STAT_NOTE_PARAM_DECL const char *_file, \ + const char *_function, \ + int _line + +#define __COMMA_LOCK_STAT_FN_DECL , const char *_function +#define __COMMA_LOCK_STAT_FN_VAR , _function +#define __COMMA_LOCK_STAT_NOTE_FN , __FUNCTION__ + +#define __COMMA_LOCK_STAT_NOTE , LOCK_STAT_NOTE +#define __COMMA_LOCK_STAT_NOTE_VARS , LOCK_STAT_NOTE_VARS +#define __COMMA_LOCK_STAT_NOTE_PARAM_DECL , LOCK_STAT_NOTE_PARAM_DECL + + +#define __COMMA_LOCK_STAT_NOTE_FLLN_DECL , const char *_file, int _line +#define __COMMA_LOCK_STAT_NOTE_FLLN , __FILE__, __LINE__ +#define __COMMA_LOCK_STAT_NOTE_FLLN_VARS , _file, _line + +#define __COMMA_LOCK_STAT_INITIALIZER , .lock_stat = NULL, + +#define __COMMA_LOCK_STAT_IP_DECL , unsigned long _ip +#define __COMMA_LOCK_STAT_IP , _ip +#define __COMMA_LOCK_STAT_RET_IP , (unsigned long) \ + __builtin_return_address(0) + +extern void lock_stat_init(struct lock_stat *ls); +extern void lock_stat_sys_init(void); + +#define lock_stat_is_initialized(o) ((unsigned long) (*o)->file) + +extern void lock_stat_note(lock_stat_ref_t *ls, struct task_struct *owner, + unsigned long ip, + int handoff); +extern void lock_stat_print(void); +extern int lock_stat_scoped_attach(lock_stat_ref_t *_s, + LOCK_STAT_NOTE_PARAM_DECL); + +#define ksym_strcmp(a, b) strncmp(a, b, KSYM_NAME_LEN) +#define ksym_strcpy(a, b) strncpy(a, b, KSYM_NAME_LEN) +#define ksym_strlen(a) strnlen(a, KSYM_NAME_LEN) + +/* +static inline char * ksym_strdup(const char *a) +{ + char *s = (char *) kmalloc(ksym_strlen(a), GFP_KERNEL); + return strncpy(s, a, KSYM_NAME_LEN); +} +*/ + +#define LS_INIT(name, h) { \ + /*.function,*/ .file = h, .line = 1, \ + .nhandoff = ATOMIC_INIT(0), .ntracked = 0, \ + .ncontended = ATOMIC_INIT(0), \ + .list_head = LIST_HEAD_INIT(name.list_head), \ + .rb_node.rb_left = NULL, \ + .rb_node.rb_left = NULL } \ + +#define DECLARE_LS_ENTRY(name) \ + extern struct lock_stat _lock_stat_##name##_entry + +/* char _##name##_string[] = #name; \ +*/ + +#define DEFINE_LS_ENTRY(name) \ + struct lock_stat _lock_stat_##name##_entry = \ + LS_INIT(_lock_stat_##name##_entry, #name "_string") + +DECLARE_LS_ENTRY(d_alloc); +DECLARE_LS_ENTRY(eventpoll_init_file); +/* +DECLARE_LS_ENTRY(get_empty_filp); +DECLARE_LS_ENTRY(init_once_1); +DECLARE_LS_ENTRY(init_once_2); +DECLARE_LS_ENTRY(inode_init_once_1); +DECLARE_LS_ENTRY(inode_init_once_2); +DECLARE_LS_ENTRY(inode_init_once_3); +DECLARE_LS_ENTRY(inode_init_once_4); +DECLARE_LS_ENTRY(inode_init_once_5); +DECLARE_LS_ENTRY(inode_init_once_6); +DECLARE_LS_ENTRY(inode_init_once_7); +*/ + +#else /* CONFIG_LOCK_STAT */ + +#define __COMMA_LOCK_STAT_FN_DECL +#define __COMMA_LOCK_STAT_FN_VAR +#define __COMMA_LOCK_STAT_NOTE_FN + +#define __COMMA_LOCK_STAT_NOTE_FLLN_DECL +#define __COMMA_LOCK_STAT_NOTE_FLLN +#define __COMMA_LOCK_STAT_NOTE_FLLN_VARS + +#define __COMMA_LOCK_STAT_INITIALIZER + +#define __COMMA_LOCK_STAT_IP_DECL +#define __COMMA_LOCK_STAT_IP +#define __COMMA_LOCK_STAT_RET_IP + +#endif /* CONFIG_LOCK_STAT */ + +#endif /* LOCK_STAT_H */ + ============================================================ --- kernel/lock_stat.c 6e4552eba62120704bbe9e785433d3716dd7b0c2 +++ kernel/lock_stat.c 6e4552eba62120704bbe9e785433d3716dd7b0c2 @@ -0,0 +1,643 @@ +/* + * lock_stat.h + * + * By Bill Huey (hui) billh@gnuppy.monkey.org + * Release under GPL license compatible with the Linux kernel + * + * Tue Sep 5 17:27:48 PDT 2006 + * Started after thinking about the problem the of Burning Man 2006 and lock + * lifetimes, scoping of those objects, etc... + * + * Thu Sep 14 04:01:26 PDT 2006 + * Some of this elaborate list handling stuff might not be necessary since I + * can attach all of the spinlocks at spin_lock_init() time, etc... It can be + * done out of line from the contention event. It's just a matter of how to + * detect and record contentions for spinlocks are statically initialized + * being the last part of what I need get this all going. I thought about that + * last night after going to BaG and talking to Mitch a bunch about this. + * + * Fri Sep 15 04:27:47 PDT 2006 + * I maybe have greatly simplified this today during the drive down here to + * SD. Keep all of the statically defined stuff late, but protect the + * persistent list by a simple spinlock and append it to the list immediately. + * This is possible because the static initializer stuff handles proper insert + * of the lock_stat struct during calls to spin_lock_init() for all other + * insertion cases. + * + * Special thanks go to Zwane and Peter for helping me with an initial + * implemention using RCU and lists even though that's now been replaced by + * something that's better and much simpler. + * + * Thu Sep 21 23:38:48 PDT 2006 + * I'm in San Diego this last week or so and I've been auditing the kernel + * for spinlock and rwlocks to see if they are statically defined or scoped. + * I finished that audit last night. + * + * Mon Sep 25 21:49:43 PDT 2006 + * Back in SF as of last night. Think about what I need to do to get rudimentary + * testing in place so that I know this code isn't going to crash the kernel. + * + * Fri Nov 24 19:06:31 PST 2006 + * As suggested by peterz, I removed the init_wait_queue stuff since lockdep + * already apparently has handled it. + */ + +#include +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +static DEFINE_RAW_SPINLOCK(free_store_lock); +static LIST_HEAD(lock_stat_free_store); + +static DEFINE_RAW_SPINLOCK(object_store_lock); +static LIST_HEAD(lock_stat_object_store); + +static char null_string[] = ""; +static char special_static_string[] = "-"; + +struct lock_stat _lock_stat_null_entry = + LS_INIT(_lock_stat_null_entry, null_string); + +EXPORT_SYMBOL(_lock_stat_null_entry); + +static DEFINE_LS_ENTRY(inline); /* lock_stat_inline_entry */ +static DEFINE_LS_ENTRY(untracked); /* lock_stat_untracked_entry */ +static DEFINE_LS_ENTRY(preinit); /* lock_stat_preinit_entry */ + +/* To be use for avoiding the dynamic attachment of spinlocks at runtime + * by attaching it inline with the lock initialization function */ + +DEFINE_LS_ENTRY(d_alloc); +EXPORT_SYMBOL(_lock_stat_d_alloc_entry); + +DEFINE_LS_ENTRY(eventpoll_init_file); +EXPORT_SYMBOL(_lock_stat_eventpoll_init_file_entry); + +/* +static DEFINE_LS_ENTRY(__pte_alloc); +static DEFINE_LS_ENTRY(get_empty_filp); +static DEFINE_LS_ENTRY(init_waitqueue_head); +static DEFINE_LS_ENTRY(init_buffer_head_1); +static DEFINE_LS_ENTRY(init_buffer_head_2); +static DEFINE_LS_ENTRY(init_once_1); +static DEFINE_LS_ENTRY(init_once_2); +static DEFINE_LS_ENTRY(inode_init_once_1); +static DEFINE_LS_ENTRY(inode_init_once_2); +static DEFINE_LS_ENTRY(inode_init_once_3); +static DEFINE_LS_ENTRY(inode_init_once_4); +static DEFINE_LS_ENTRY(inode_init_once_5); +static DEFINE_LS_ENTRY(inode_init_once_6); +static DEFINE_LS_ENTRY(inode_init_once_7); +static DEFINE_LS_ENTRY(inode_init_once_8); +static DEFINE_LS_ENTRY(mm_init_1); +static DEFINE_LS_ENTRY(mm_init_2); +static DEFINE_LS_ENTRY(mm_init_3); +static DEFINE_LS_ENTRY(skb_queue_init); +static DEFINE_LS_ENTRY(tcp_init_1); +static DEFINE_LS_ENTRY(tcp_init_2); +*/ + +/* + * I should never have to create more entries that this since I audited the + * kernel and found out that there are only ~1500 or so places in the kernel + * where these rw/spinlocks are initialized. Use the initialization points as a + * hash value to look up the backing objects + */ + +#define MAGIC_ENTRIES 1600 + +static int lock_stat_procfs_init(void); +static void lock_stat_insert_object(struct lock_stat *); + +static int lock_stat_inited = 0; + +static +DEFINE_PER_CPU(atomic_t, lock_stat_total_events); + +static +struct rb_root lock_stat_rbtree_db = RB_ROOT; + +static void lock_stat_rbtree_db_print(struct seq_file *seq); +static void _lock_stat_rbtree_db_print(struct seq_file *seq, + struct rb_node *node, + int level); + +static void lock_stat_rbtree_db_zero(void); +static void _lock_stat_rbtree_db_zero(struct rb_node *node); + +static +void lock_stat_rbtree_db_zero(void) +{ + _lock_stat_rbtree_db_zero(lock_stat_rbtree_db.rb_node); +} + +void lock_stat_init(struct lock_stat *oref) +{ + oref->function[0] = 0; + oref->file = NULL; + oref->line = 0; + + oref->ntracked = 0; + atomic_set(&oref->ninlined, 0); + atomic_set(&oref->ncontended, 0); + atomic_set(&oref->nspinnable, 0); + atomic_set(&oref->nhandoff, 0); + + INIT_LIST_HEAD(&oref->list_head); + oref->rb_node.rb_left = oref->rb_node.rb_right = NULL; +} + +void lock_stat_reset_stats(void) +{ + int cpu; + + lock_stat_rbtree_db_zero(); + + for_each_possible_cpu(cpu) { + atomic_set(&per_cpu(lock_stat_total_events, cpu), 0); + } +} + +void lock_stat_sys_init(void) +{ + struct lock_stat *s; + int i; + + for (i = 0; i < MAGIC_ENTRIES; ++i) { + s = kmalloc(sizeof(struct lock_stat), GFP_KERNEL); + + if (s) { + lock_stat_init(s); + list_add_tail(&s->list_head, &lock_stat_free_store); + } + else { + printk("%s: kmalloc returned NULL\n", __func__); + return; + } + } + + lock_stat_insert_object(&_lock_stat_inline_entry); + lock_stat_insert_object(&_lock_stat_untracked_entry); + lock_stat_insert_object(&_lock_stat_preinit_entry); + + lock_stat_insert_object(&_lock_stat_d_alloc_entry); + lock_stat_insert_object(&_lock_stat_eventpoll_init_file_entry); + + lock_stat_procfs_init(); + + lock_stat_reset_stats(); + lock_stat_inited = 1; +} + +static +struct lock_stat *lock_stat_allocate_object(void) +{ + unsigned long flags; + + spin_lock_irqsave(&free_store_lock, flags); + if (!list_empty(&lock_stat_free_store)) { + struct list_head *e = lock_stat_free_store.next; + struct lock_stat *s; + + s = container_of(e, struct lock_stat, list_head); + list_del(e); + + spin_unlock_irqrestore(&free_store_lock, flags); + + return s; + } + spin_unlock_irqrestore(&free_store_lock, flags); + + panic("%s: out of preallocated objects\n", __func__); + + return NULL; +} + +/* + * For ordered insertion into a tree. The (entry - object) order of + * comparison is switched from the parameter definition. + */ +static +int lock_stat_compare_objs(struct lock_stat *x, struct lock_stat *y) +{ + int a = 0, b = 0, c = 0; + + (a = ksym_strcmp(x->function, y->function)) || + (b = ksym_strcmp(x->file, y->file)) || + (c = (x->line - y->line)); + + return a | b | c; +} + +static +int lock_stat_compare_key_to_obj(LOCK_STAT_NOTE_PARAM_DECL, struct lock_stat *x) +{ + int a = 0, b = 0, c = 0; + + (a = ksym_strcmp(_function, x->function)) || + (b = ksym_strcmp(_file, x->file)) || + (c = (_line - x->line)); + + return a | b | c; +} + +static +void lock_stat_print_entry(struct seq_file *seq, struct lock_stat *o) +{ + seq_printf(seq, "[%d, %d, %d - %d, %d]\t\t{%s, %s, %d}\n", + atomic_read(&o->ncontended), + atomic_read(&o->nspinnable), + atomic_read(&o->nhandoff), + o->ntracked, + atomic_read(&o->ninlined), + o->function, + o->file, + o->line + ); +} + +static +void _lock_stat_rbtree_db_zero(struct rb_node *node) +{ + struct lock_stat *o = container_of(node, struct lock_stat, rb_node); + + if (!node) + return; + + _lock_stat_rbtree_db_zero(node->rb_left); + atomic_set(&o->ncontended, 0); + _lock_stat_rbtree_db_zero(node->rb_right); +} + +static +void lock_stat_rbtree_db_print(struct seq_file *seq) +{ + _lock_stat_rbtree_db_print(seq, lock_stat_rbtree_db.rb_node, 0); +} + +static int missed = 0, attached = 0, left_insert = 0, right_insert = 0, found = 0; + +static +void _lock_stat_rbtree_db_print(struct seq_file *seq, struct rb_node *node, + int level) +{ + struct lock_stat *o = container_of(node, struct lock_stat, rb_node); + + if (!node) + return; + + ++level; + + _lock_stat_rbtree_db_print(seq, node->rb_left, level); + lock_stat_print_entry(seq, o); + _lock_stat_rbtree_db_print(seq, node->rb_right, level); +} + +static +struct lock_stat *lock_stat_insert_rbtree_db(struct lock_stat *o) +{ + struct rb_node **p = &lock_stat_rbtree_db.rb_node; + struct rb_node *parent = NULL; + struct lock_stat *cursor; + + while (*p) { + parent = *p; + cursor = container_of(parent, struct lock_stat, rb_node); + + if (lock_stat_compare_objs(o, cursor) < 0) { + p = &(*p)->rb_left; + ++left_insert; + } + else if (lock_stat_compare_objs(o, cursor) > 0) { + p = &(*p)->rb_right; + ++right_insert; + } + else { /* means we found a duplicate */ + ++missed; + return o; + } + } + + rb_link_node(&o->rb_node, parent, p); + rb_insert_color(&o->rb_node, &lock_stat_rbtree_db); + + ++attached; + + return NULL; +} + +static +struct lock_stat *lock_stat_lookup_rbtree_db(LOCK_STAT_NOTE_PARAM_DECL) +{ + struct rb_node *node = lock_stat_rbtree_db.rb_node; + struct lock_stat *cursor; + + while (node) + { + cursor = container_of(node, struct lock_stat, rb_node); + /* + * item less than-> left + * item great than -> right + * + * The relationship of lock_stat_obj_compare() parameter is + * reversed so reverse the comparison as well. + */ + if (lock_stat_compare_key_to_obj(LOCK_STAT_NOTE_VARS, cursor) < 0) + node = node->rb_left; + else if (lock_stat_compare_key_to_obj(LOCK_STAT_NOTE_VARS, cursor) > 0) + node = node->rb_right; + else + return cursor; + } + return NULL; +} + +/* must hold object_store_lock */ +static +struct lock_stat * lock_stat_lookup_object(LOCK_STAT_NOTE_PARAM_DECL) +{ + return lock_stat_lookup_rbtree_db(LOCK_STAT_NOTE_VARS); +} + +/* Must hold object_store_lock */ +static +void lock_stat_insert_object(struct lock_stat *o) +{ + lock_stat_insert_rbtree_db(o); +} + +/* + * (1) either immediately allocate or link the backing object at a + * spin_lock_init() call. + * + * For rtmutexes that are statically scoped, there is a only one occurance of + * a mutex that exists and you only have to worry about one backing object for + * that instance. Blindly adding this to the lock_stat dictionary is ok since + * this happens only once and this should have little effect on the overall + * system. It's a one time operation at the first contention. + * + * For cases where a rtmutex is dynamically allocate and there multipule + * instances, you still track it with one backing object but it's connect to + * this object by the initialization point (file, function, line) of the + * rtmutex. That place in the source file is also used as an identifying key + * for that instance and it is used as to associate it with a backing statistical + * object. They are forever connected together from that point by that key. That + * backing object holds the total number of contentions and other stats for all + * objects associated with that key. + * + * There maybe other initialization points for that same structure effecting + * how the keys are utilized, but they should * all effectively be connected + * to a single lock_stat object representing the * overall contention behavior + * and use of that object. + * + * Connecting the rtmutex to the lock_stat object at spin_lock_init() time. It's + * not a critical path. There are cases where a C99 style struct initializer is + * used so I can't insert it into the lock_stat dictionary and those initializer + * must be converted to use spin_lock_init() instead. + * + */ +int lock_stat_scoped_attach(lock_stat_ref_t *oref, LOCK_STAT_NOTE_PARAM_DECL) +{ + unsigned long flags; + struct lock_stat *o = NULL; + + /* failed error condition */ + if (!oref) + return 0; + + /* ignore instrumentation calls before it's intialized */ + if (!lock_stat_inited) + return 0; + + /* Dump all stats into a default entry for wacked out keys */ + if (_file == NULL || _function == NULL) { + *oref = &_lock_stat_null_entry; + return 0; + } + + spin_lock_irqsave(&object_store_lock, flags); + + /* If it doesn't exist, make the backing object and insert it + * into the dictionary */ + + o = lock_stat_lookup_object(LOCK_STAT_NOTE_VARS); + + if (!o) { + o = lock_stat_allocate_object(); + + DEBUG_LOCKS_WARN_ON(!o); + + ksym_strcpy(o->function, _function); + o->file = (char *)_file; + o->line = _line; + + *oref = o; + lock_stat_insert_object(o); + } else { + *oref = o; + } + + + ++(*oref)->ntracked; + spin_unlock_irqrestore(&object_store_lock, flags); + + return 0; +} + + +/* + * (2) This is for a late detection case for statically allocated spinlock + * structures. + */ + +static +void lock_stat_static_attach(lock_stat_ref_t *oref, LOCK_STAT_NOTE_PARAM_DECL) +{ + unsigned long flags; + struct lock_stat *o, *p; + + DEBUG_LOCKS_WARN_ON(!oref); + + if (_file == NULL || _function == NULL) { + *oref = &_lock_stat_null_entry; + return; + } + + spin_lock_irqsave(&object_store_lock, flags); + + /* try it again, just in case */ + if (*oref) + goto out; + + if ((p = lock_stat_lookup_object(LOCK_STAT_NOTE_VARS))) + ++found; + + o = lock_stat_allocate_object(); + + DEBUG_LOCKS_WARN_ON(!o); + + *oref = o; + ++((*oref)->ntracked); + + ksym_strcpy(o->function, _function); + o->file = (char *) _file; + o->line = _line; + + lock_stat_insert_object(o); + +out: + spin_unlock_irqrestore(&object_store_lock, flags); +} + +static +char *ksym_get(unsigned long address, char *namebuffer) +{ + unsigned long offset = 0, symsize; + char *modname; + const char *symname; + + symname = kallsyms_lookup(address, &symsize, &offset, &modname, + namebuffer); + + if (!symname) { + return null_string; + } + + return (char *) symname; +} + +extern int task_on_rq(int cpu, struct task_struct *p); + +void lock_stat_note(lock_stat_ref_t *oref, struct task_struct *owner, + unsigned long ip, + int handoff) +{ + int cpu; + char ksym_scoped_namebuf[KSYM_NAME_LEN+1]; + extern struct task_struct *get_task_on_rq(int cpu); + + DEBUG_LOCKS_WARN_ON(!oref); + + if (unlikely(!lock_stat_inited)) + return; + + ksym_scoped_namebuf[KSYM_NAME_LEN] = '\0'; + + if (unlikely(!(*oref))) { + char *r = NULL; + /* + * Case (2) for statically defined rtmutex + * + * Fri Oct 27 00:26:08 PDT 2006 + * This is for the cases where a statically defined lock is passed + * to a structure that's dynamically allocated. I didn't know that + * the kernel did this until today looking at the function + * as_work_handler() + */ + + r = ksym_get(ip, ksym_scoped_namebuf); + DEBUG_LOCKS_WARN_ON(strnlen(ksym_scoped_namebuf, KSYM_NAME_LEN) >= KSYM_NAME_LEN); + + lock_stat_static_attach(oref, special_static_string, + ksym_scoped_namebuf, + 0); + } + + DEBUG_LOCKS_WARN_ON(!oref); + + cpu = task_cpu(owner); + + if (handoff == LOCK_STAT_NOTE_TYPE_HANDOFF) { + atomic_inc(&((*oref)->nhandoff)); + } + else { + if (owner && (owner == get_task_on_rq(cpu))) + atomic_inc(&((*oref)->nspinnable)); + + atomic_inc(&((*oref)->ncontended)); + } + + atomic_inc(&per_cpu(lock_stat_total_events, get_cpu())); + put_cpu(); +} + +static int lock_stat_procfs_open(struct inode *, struct file *); +static int lock_stat_procfs_write(struct file *file, const char *buf, + size_t count, + loff_t *off); + +struct proc_dir_entry *lock_stat_procfs_dir; +struct proc_dir_entry *lock_stat_procfs_entry; + +static struct file_operations lock_stat_procfs_ops = { + .open = lock_stat_procfs_open, + .read = seq_read, + .write = lock_stat_procfs_write, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int lock_stat_procfs_init(void) +{ + int error = 0; + + lock_stat_procfs_dir = proc_mkdir("lock_stat", NULL); + if (lock_stat_procfs_dir == NULL) { + error = -ENOMEM; + return error; + } + + lock_stat_procfs_entry = create_proc_entry("contention", 0666, + lock_stat_procfs_dir); + if (lock_stat_procfs_entry == NULL) { + error = -ENOMEM; + return error; + } + else { + lock_stat_procfs_entry->proc_fops = &lock_stat_procfs_ops; + } + + return 0; +} + +static int lock_stat_procfs_show(struct seq_file *seq, void *v) +{ + int cpu, count = 0; + + for_each_possible_cpu(cpu) { + count += atomic_read(&per_cpu(lock_stat_total_events, cpu)); + } + + seq_printf(seq, "@contention events = %d\n", count); + seq_printf(seq, "@found = %d\n", found); + + lock_stat_rbtree_db_print(seq); + return 0; +} + +static int lock_stat_procfs_write (struct file *file, const char *buf, + size_t count, + loff_t *off) +{ + lock_stat_reset_stats(); + return count; +} + +static int lock_stat_procfs_open(struct inode *inode, struct file *file) { + return single_open (file, &lock_stat_procfs_show, NULL ); +} + ============================================================ --- arch/xtensa/platform-iss/network.c 8c372aaff21fb65a0a03f62d19d53c1d0b1cefb8 +++ arch/xtensa/platform-iss/network.c a60ede200393383d23f4d4e5ade70a6b94f580ca @@ -648,6 +648,8 @@ static int iss_net_configure(int index, .have_mac = 0, }); + spin_lock_init(&lp->lock); + /* * Try all transport protocols. * Note: more protocols can be added by adding '&& !X_init(lp, eth)'. ============================================================ --- fs/dcache.c ee722526acdb0b5e749dbe54d836216e5e0c696a +++ fs/dcache.c b01158646d18e7a951c083cfad774f5628fbcfdf @@ -892,7 +892,7 @@ struct dentry *d_alloc(struct dentry * p atomic_set(&dentry->d_count, 1); dentry->d_flags = DCACHE_UNHASHED; - spin_lock_init(&dentry->d_lock); + spin_lock_init_annotated(&dentry->d_lock, &_lock_stat_d_alloc_entry); dentry->d_inode = NULL; dentry->d_parent = NULL; dentry->d_sb = NULL; ============================================================ --- include/linux/eventpoll.h bd142a622609d04952fac6215586fff353dab729 +++ include/linux/eventpoll.h 43271ded1a3b9f40beb37aaff9e02fadeecb4655 @@ -15,6 +15,7 @@ #define _LINUX_EVENTPOLL_H #include +#include /* Valid opcodes to issue to sys_epoll_ctl() */ @@ -55,7 +56,7 @@ static inline void eventpoll_init_file(s static inline void eventpoll_init_file(struct file *file) { INIT_LIST_HEAD(&file->f_ep_links); - spin_lock_init(&file->f_ep_lock); + spin_lock_init_annotated(&file->f_ep_lock, &_lock_stat_eventpoll_init_file_entry); } ============================================================ --- include/linux/mutex.h 8d322d0e6ab629c98d1222fedf1d6e8ab0afbc31 +++ include/linux/mutex.h 1d77e157da8b1716940135dfe86ce97ea2fd8d17 @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -35,7 +36,8 @@ extern void } extern void -_mutex_init(struct mutex *lock, char *name, struct lock_class_key *key); +_mutex_init(struct mutex *lock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL); extern void __lockfunc _mutex_lock(struct mutex *lock); extern int __lockfunc _mutex_lock_interruptible(struct mutex *lock); @@ -61,11 +63,15 @@ extern void __lockfunc _mutex_unlock(str _mutex_lock_interruptible(l) #endif +#define __mutex_init(l,n) __rt_mutex_init(&(l)->mutex, \ + n \ + __COMMA_LOCK_STAT_NOTE) + # define mutex_init(mutex) \ do { \ static struct lock_class_key __key; \ \ - _mutex_init((mutex), #mutex, &__key); \ + _mutex_init((mutex), #mutex, &__key __COMMA_LOCK_STAT_NOTE); \ } while (0) #else ============================================================ --- include/linux/rt_lock.h e257892e9eb655e7bd12ed00ff23f5167fb97b5d +++ include/linux/rt_lock.h 420425e6ce33342d9f5eb6b2e4be80453b7139cc @@ -13,6 +13,7 @@ #include #include #include +#include #ifdef CONFIG_PREEMPT_RT /* @@ -29,10 +30,13 @@ typedef struct { #ifdef CONFIG_DEBUG_RT_MUTEXES # define __SPIN_LOCK_UNLOCKED(name) \ (spinlock_t) { { .wait_lock = _RAW_SPIN_LOCK_UNLOCKED(name) \ - , .save_state = 1, .file = __FILE__, .line = __LINE__ }, SPIN_DEP_MAP_INIT(name) } + , .save_state = 1, .file = __FILE__, .line = __LINE__ \ + __COMMA_LOCK_STAT_INITIALIZER}, \ + SPIN_DEP_MAP_INIT(name) } #else # define __SPIN_LOCK_UNLOCKED(name) \ - (spinlock_t) { { .wait_lock = _RAW_SPIN_LOCK_UNLOCKED(name) }, SPIN_DEP_MAP_INIT(name) } + (spinlock_t) { { .wait_lock = _RAW_SPIN_LOCK_UNLOCKED(name) }, \ + SPIN_DEP_MAP_INIT(name) } #endif # define SPIN_LOCK_UNLOCKED __SPIN_LOCK_UNLOCKED(spin_old_style) #else /* !PREEMPT_RT */ @@ -92,7 +96,8 @@ typedef struct { # ifdef CONFIG_DEBUG_RT_MUTEXES # define __RW_LOCK_UNLOCKED(name) (rwlock_t) \ { .lock = { .wait_lock = _RAW_SPIN_LOCK_UNLOCKED(name), \ - .save_state = 1, .file = __FILE__, .line = __LINE__ } } + .save_state = 1, .file = __FILE__, .line = __LINE__ + __COMMA_LOCK_STAT_INITIALIZER } } # else # define __RW_LOCK_UNLOCKED(name) (rwlock_t) \ { .lock = { .wait_lock = _RAW_SPIN_LOCK_UNLOCKED(name) } } @@ -139,14 +144,22 @@ struct semaphore name = \ */ #define DECLARE_MUTEX_LOCKED COMPAT_DECLARE_MUTEX_LOCKED -extern void fastcall __sema_init(struct semaphore *sem, int val, char *name, char *file, int line); +extern void fastcall __sema_init(struct semaphore *sem, int val, char *name + __COMMA_LOCK_STAT_FN_DECL, + char *_file, + int _line); #define rt_sema_init(sem, val) \ - __sema_init(sem, val, #sem, __FILE__, __LINE__) + __sema_init(sem, val, #sem __COMMA_LOCK_STAT_NOTE_FN, \ + __FILE__, \ + __LINE__) -extern void fastcall __init_MUTEX(struct semaphore *sem, char *name, char *file, int line); +extern void fastcall __init_MUTEX(struct semaphore *sem, char *name + __COMMA_LOCK_STAT_FN_DECL, char *_file, int _line); #define rt_init_MUTEX(sem) \ - __init_MUTEX(sem, #sem, __FILE__, __LINE__) + __init_MUTEX(sem, #sem __COMMA_LOCK_STAT_NOTE_FN, \ + __FILE__, \ + __LINE__) extern void there_is_no_init_MUTEX_LOCKED_for_RT_semaphores(void); @@ -247,13 +260,14 @@ extern void fastcall __rt_rwsem_init(str struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname) extern void fastcall __rt_rwsem_init(struct rw_semaphore *rwsem, char *name, - struct lock_class_key *key); + struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL); # define rt_init_rwsem(sem) \ do { \ static struct lock_class_key __key; \ \ - __rt_rwsem_init((sem), #sem, &__key); \ + __rt_rwsem_init((sem), #sem, &__key __COMMA_LOCK_STAT_NOTE); \ } while (0) extern void fastcall rt_down_write(struct rw_semaphore *rwsem); ============================================================ --- include/linux/rtmutex.h e6fa10297e6c20d27edba172aeb078a60c64488e +++ include/linux/rtmutex.h 55cd2de44a52e049fa8a0da63bde6449cefeb8fe @@ -15,6 +15,7 @@ #include #include #include +#include /* * The rt_mutex structure @@ -33,6 +34,9 @@ struct rt_mutex { int line; void *magic; #endif +#ifdef CONFIG_LOCK_STAT + struct lock_stat *lock_stat; +#endif }; struct rt_mutex_waiter; @@ -54,11 +58,13 @@ struct hrtimer_sleeper; #ifdef CONFIG_DEBUG_RT_MUTEXES # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) \ , .name = #mutexname, .file = __FILE__, .line = __LINE__ -# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __FUNCTION__) +# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __FUNCTION__ \ + __COMMA_LOCK_STAT_NOTE_DECL_FLLN) extern void rt_mutex_debug_task_free(struct task_struct *tsk); #else # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) -# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL) +# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL \ + __COMMA_LOCK_STAT_NOTE_DECL_FLLN) # define rt_mutex_debug_task_free(t) do { } while (0) #endif @@ -66,6 +72,7 @@ struct hrtimer_sleeper; { .wait_lock = RAW_SPIN_LOCK_UNLOCKED(mutexname) \ , .wait_list = PLIST_HEAD_INIT(mutexname.wait_list, mutexname.wait_lock) \ , .owner = NULL \ + __COMMA_LOCK_STAT_INITIALIZER \ __DEBUG_RT_MUTEX_INITIALIZER(mutexname)} #define DEFINE_RT_MUTEX(mutexname) \ @@ -82,10 +89,19 @@ static inline int rt_mutex_is_locked(str return lock->owner != NULL; } -extern void __rt_mutex_init(struct rt_mutex *lock, const char *name); +extern void __rt_mutex_init(struct rt_mutex *lock, + const char *name + __COMMA_LOCK_STAT_NOTE_PARAM_DECL); +#ifdef CONFIG_LOCK_STAT +extern void __rt_mutex_init_annotated(struct rt_mutex *lock, const char *name, + LOCK_STAT_NOTE_PARAM_DECL, + struct lock_stat *lsobject); +#endif + extern void rt_mutex_destroy(struct rt_mutex *lock); extern void rt_mutex_lock(struct rt_mutex *lock); + extern int rt_mutex_lock_interruptible(struct rt_mutex *lock, int detect_deadlock); extern int rt_mutex_timed_lock(struct rt_mutex *lock, @@ -96,6 +112,20 @@ extern void rt_mutex_unlock(struct rt_mu extern void rt_mutex_unlock(struct rt_mutex *lock); +#ifdef CONFIG_LOCK_STAT +extern void rt_mutex_lock_with_ip(struct rt_mutex *lock, + unsigned long ip); +extern int rt_mutex_lock_interruptible_with_ip(struct rt_mutex *lock, + int detect_deadlock, + unsigned long ip); +extern int rt_mutex_timed_lock_with_ip(struct rt_mutex *lock, + struct hrtimer_sleeper *timeout, + int detect_deadlock, + unsigned long ip); +extern int rt_mutex_trylock_with_ip(struct rt_mutex *lock, + unsigned long ip); +#endif + #ifdef CONFIG_RT_MUTEXES # define INIT_RT_MUTEXES(tsk) \ .pi_waiters = PLIST_HEAD_INIT(tsk.pi_waiters, tsk.pi_lock), \ ============================================================ --- include/linux/spinlock.h b4a4e821bac27625019dd765523d82f6e131f4b0 +++ include/linux/spinlock.h 144f383da63fc13b331bb5459fb66528c20e569c @@ -91,6 +91,7 @@ #include #include #include +#include #include @@ -153,8 +154,17 @@ extern void extern int __bad_rwlock_type(void); extern void -__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key); +__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL); +#ifdef CONFIG_LOCK_STAT +extern void +__rt_spin_lock_init_annotated(spinlock_t *lock, char *name, + struct lock_class_key *key, + LOCK_STAT_NOTE_PARAM_DECL, + struct lock_stat *lsobject); +#endif + extern void __lockfunc rt_spin_lock(spinlock_t *lock); extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass); extern void __lockfunc rt_spin_unlock(spinlock_t *lock); @@ -171,6 +181,11 @@ extern void __lockfunc __rt_spin_unlock( extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock); extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock); +#ifdef CONFIG_LOCK_STAT +extern void __lockfunc __rt_spin_lock_with_ip(struct rt_mutex *lock, + unsigned long _ip); +#endif + #ifdef CONFIG_PREEMPT_RT # define _spin_lock(l) rt_spin_lock(l) # define _spin_lock_nested(l, s) rt_spin_lock_nested(l, s) @@ -219,9 +234,18 @@ do { \ do { \ static struct lock_class_key __key; \ \ - __rt_spin_lock_init(sl, n, &__key); \ + __rt_spin_lock_init(sl, n, &__key __COMMA_LOCK_STAT_NOTE); \ } while (0) +#ifdef CONFIG_LOCK_STAT +#define _spin_lock_init_annotated(sl, n, f, l, lsobj) \ +do { \ + static struct lock_class_key __key; \ + \ + __rt_spin_lock_init_annotated(sl, n, &__key, f, __func__, l, lsobj); \ +} while (0) +#endif + # ifdef CONFIG_PREEMPT_RT # define _spin_can_lock(l) (!rt_mutex_is_locked(&(l)->lock)) # define _spin_is_locked(l) rt_mutex_is_locked(&(l)->lock) @@ -302,15 +326,31 @@ extern void extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock); extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock); extern void -__rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key); +__rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL); #define _rwlock_init(rwl, n, f, l) \ do { \ static struct lock_class_key __key; \ \ - __rt_rwlock_init(rwl, n, &__key); \ + __rt_rwlock_init(rwl, n, &__key __COMMA_LOCK_STAT_NOTE); \ } while (0) +#ifdef CONFIG_LOCK_STAT +extern void +__rt_rwlock_init_annotated(rwlock_t *rwlock, char *name, + struct lock_class_key *key, + LOCK_STAT_NOTE_PARAM_DECL, + struct lock_stat *lsobject); + +#define _rwlock_init_annotated(rwl, n, f, l, lsobject) \ +do { \ + static struct lock_class_key __key; \ + \ + __rt_rwlock_init(rwl, n, &__key, LOCK_STAT_NOTE, lsobject); \ +} while (0) +#endif + #ifdef CONFIG_PREEMPT_RT # define rt_read_can_lock(rwl) (!rt_mutex_is_locked(&(rwl)->lock)) # define rt_write_can_lock(rwl) (!rt_mutex_is_locked(&(rwl)->lock)) @@ -428,6 +468,25 @@ do { \ #define spin_lock_init(lock) PICK_OP_INIT(_lock_init, lock) +#ifdef CONFIG_LOCK_STAT +#define PICK_OP_INIT_ANNOTATED(op, lock, lsobj) \ +do { \ + if (TYPE_EQUAL((lock), raw_spinlock_t)) \ + _raw_spin##op((raw_spinlock_t *)(lock)); \ + else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin##op##_annotated((spinlock_t *)(lock), #lock, \ + __FILE__, \ + __LINE__, \ + lsobj); \ + else __bad_spinlock_type(); \ +} while (0) + +#define spin_lock_init_annotated(lock, lsobj) PICK_OP_INIT_ANNOTATED( \ + _lock_init, \ + lock, \ + lsobj) +#endif + #ifdef CONFIG_DEBUG_SPINLOCK extern void __raw_rwlock_init(raw_rwlock_t *lock, const char *name, struct lock_class_key *key); ============================================================ --- include/linux/wait.h 12da8de69f1f2660443a04c3df199e5d851ea2ca +++ include/linux/wait.h 9b7448af82583bd11d18032aedfa8f2af44345f4 @@ -81,7 +81,7 @@ extern void init_waitqueue_head(wait_que extern void init_waitqueue_head(wait_queue_head_t *q); -#ifdef CONFIG_LOCKDEP +#if defined(CONFIG_LOCKDEP) || defined(CONFIG_LOCK_STAT) # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \ ({ init_waitqueue_head(&name); name; }) # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \ ============================================================ --- init/main.c aac66f499d9b75035c984a599d00646fe9b81114 +++ init/main.c b0c6e65355a4c13f28fa91b9449795045aa21e93 @@ -621,6 +621,7 @@ asmlinkage void __init start_kernel(void #ifdef CONFIG_PROC_FS proc_root_init(); #endif + lock_stat_sys_init(); cpuset_init(); taskstats_init_early(); delayacct_init(); ============================================================ --- kernel/Kconfig.preempt 3148bd94270ea0a853d8e443616cd7a668dd0d3b +++ kernel/Kconfig.preempt d63831dbfbb9e68386bfc862fd2dd1a8f1e9779f @@ -176,3 +176,12 @@ config RCU_TRACE Say Y here if you want to enable RCU tracing Say N if you are unsure. + +config LOCK_STAT + bool "Lock contention statistics tracking in /proc" + depends on PREEMPT_RT && !DEBUG_RT_MUTEXES + default y + help + General lock statistics tracking with regard to contention in + /proc/lock_stat/contention + ============================================================ --- kernel/Makefile 0690fbe8c605a1c7e24b7b94d05a96ea32574aab +++ kernel/Makefile 08087775b67b7ac1682dac0310003ef7ecbd7e70 @@ -63,6 +63,7 @@ obj-$(CONFIG_TASKSTATS) += taskstats.o t obj-$(CONFIG_UTS_NS) += utsname.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o +obj-$(CONFIG_LOCK_STAT) += lock_stat.o ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) # According to Alan Modra , the -fno-omit-frame-pointer is ============================================================ --- kernel/rt.c e7b738b39985ab0afd49068c3c937dece5249f50 +++ kernel/rt.c 03449e79eedce653a4acc874f65301b17f10b2b4 @@ -66,6 +66,7 @@ #include #include #include +#include #include "rtmutex_common.h" @@ -75,6 +76,42 @@ # include "rtmutex.h" #endif +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_MUTEX_LOCK(a) \ + rt_mutex_lock_with_ip(a, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_MUTEX_LOCK(a) \ + rt_mutex_lock(a); +#endif + +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_MUTEX_LOCK_INTERRUPTIBLE(a, b) \ + rt_mutex_lock_interruptible_with_ip(a, b, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_MUTEX_LOCK_INTERRUPTIBLE(a) \ + rt_mutex_lock_interruptible(a, b); +#endif + +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_MUTEX_TRYLOCK(a) \ + rt_mutex_trylock_with_ip(a, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_MUTEX_TRYLOCK(a) \ + rt_mutex_trylock(a); +#endif + +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_SPIN_LOCK(a) \ + __rt_spin_lock_with_ip(a, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_SPIN_LOCK(a) \ + __rt_spin_lock(a); +#endif + #ifdef CONFIG_PREEMPT_RT /* * Unlock these on crash: @@ -88,7 +125,8 @@ void zap_rt_locks(void) /* * struct mutex functions */ -void _mutex_init(struct mutex *lock, char *name, struct lock_class_key *key) +void _mutex_init(struct mutex *lock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) { #ifdef CONFIG_DEBUG_LOCK_ALLOC /* @@ -97,14 +135,15 @@ void _mutex_init(struct mutex *lock, cha debug_check_no_locks_freed((void *)lock, sizeof(*lock)); lockdep_init_map(&lock->dep_map, name, key, 0); #endif - __rt_mutex_init(&lock->lock, name); + __rt_mutex_init(&lock->lock, name __COMMA_LOCK_STAT_NOTE_VARS); } EXPORT_SYMBOL(_mutex_init); void __lockfunc _mutex_lock(struct mutex *lock) { mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); - rt_mutex_lock(&lock->lock); + + __LOCK_STAT_RT_MUTEX_LOCK(&lock->lock); } EXPORT_SYMBOL(_mutex_lock); @@ -124,7 +163,7 @@ void __lockfunc _mutex_lock_nested(struc void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) { mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); - rt_mutex_lock(&lock->lock); + __LOCK_STAT_RT_MUTEX_LOCK(&lock->lock); } EXPORT_SYMBOL(_mutex_lock_nested); @@ -143,7 +182,7 @@ int __lockfunc _mutex_trylock(struct mut int __lockfunc _mutex_trylock(struct mutex *lock) { - int ret = rt_mutex_trylock(&lock->lock); + int ret = __LOCK_STAT_RT_MUTEX_TRYLOCK(&lock->lock); if (ret) mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); @@ -164,7 +203,7 @@ int __lockfunc rt_write_trylock(rwlock_t */ int __lockfunc rt_write_trylock(rwlock_t *rwlock) { - int ret = rt_mutex_trylock(&rwlock->lock); + int ret = __LOCK_STAT_RT_MUTEX_TRYLOCK(&rwlock->lock); if (ret) rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); @@ -191,7 +230,7 @@ int __lockfunc rt_read_trylock(rwlock_t } spin_unlock_irqrestore(&lock->wait_lock, flags); - ret = rt_mutex_trylock(lock); + ret = __LOCK_STAT_RT_MUTEX_TRYLOCK(lock); if (ret) rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_); @@ -202,7 +241,7 @@ void __lockfunc rt_write_lock(rwlock_t * void __lockfunc rt_write_lock(rwlock_t *rwlock) { rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); - __rt_spin_lock(&rwlock->lock); + __LOCK_STAT_RT_SPIN_LOCK(&rwlock->lock); } EXPORT_SYMBOL(rt_write_lock); @@ -222,11 +261,44 @@ void __lockfunc rt_read_lock(rwlock_t *r return; } spin_unlock_irqrestore(&lock->wait_lock, flags); - __rt_spin_lock(lock); + __LOCK_STAT_RT_SPIN_LOCK(lock); } EXPORT_SYMBOL(rt_read_lock); +#ifdef CONFIG_LOCK_STAT +void __lockfunc rt_write_lock_with_ip(rwlock_t *rwlock, unsigned long ip) +{ + rwlock_acquire(&rwlock->dep_map, 0, 0, ip); + __rt_spin_lock_with_ip(&rwlock->lock, ip); +} +EXPORT_SYMBOL(rt_write_lock_with_ip); + +void __lockfunc rt_read_lock_with_ip(rwlock_t *rwlock, unsigned long ip) +{ + unsigned long flags; + struct rt_mutex *lock = &rwlock->lock; + + /* + * NOTE: we handle it as a write-lock: + */ + rwlock_acquire(&rwlock->dep_map, 0, 0, ip); + /* + * Read locks within the write lock succeed. + */ + spin_lock_irqsave(&lock->wait_lock, flags); + if (rt_mutex_real_owner(lock) == current) { + spin_unlock_irqrestore(&lock->wait_lock, flags); + rwlock->read_depth++; + return; + } + spin_unlock_irqrestore(&lock->wait_lock, flags); + __rt_spin_lock_with_ip(lock, ip); +} + +EXPORT_SYMBOL(rt_read_lock_with_ip); +#endif + void __lockfunc rt_write_unlock(rwlock_t *rwlock) { /* NOTE: we always pass in '1' for nested, for simplicity */ @@ -258,7 +330,12 @@ unsigned long __lockfunc rt_write_lock_i unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock) { +#ifdef CONFIG_LOCK_STAT + rt_write_lock_with_ip(rwlock, + (unsigned long) __builtin_return_address(0)); +#else rt_write_lock(rwlock); +#endif return 0; } @@ -266,13 +343,19 @@ unsigned long __lockfunc rt_read_lock_ir unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock) { +#ifdef CONFIG_LOCK_STAT + rt_read_lock_with_ip(rwlock, + (unsigned long) __builtin_return_address(0)); +#else rt_read_lock(rwlock); +#endif return 0; } EXPORT_SYMBOL(rt_read_lock_irqsave); -void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key) +void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) { #ifdef CONFIG_DEBUG_LOCK_ALLOC /* @@ -281,11 +364,30 @@ void __rt_rwlock_init(rwlock_t *rwlock, debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); lockdep_init_map(&rwlock->dep_map, name, key, 0); #endif - __rt_mutex_init(&rwlock->lock, name); + __rt_mutex_init(&rwlock->lock, name __COMMA_LOCK_STAT_NOTE_VARS); rwlock->read_depth = 0; } EXPORT_SYMBOL(__rt_rwlock_init); +#ifdef CONFIG_LOCK_STAT +void __rt_rwlock_init_annotated(rwlock_t *rwlock, char *name, + struct lock_class_key *key, + LOCK_STAT_NOTE_PARAM_DECL, + struct lock_stat *lsobject) +{ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* + * Make sure we are not reinitializing a held lock: + */ + debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); + lockdep_init_map(&rwlock->dep_map, name, key, 0); +#endif + __rt_mutex_init_annotated(&rwlock->lock, name, LOCK_STAT_NOTE_VARS, lsobject); + rwlock->read_depth = 0; +} +EXPORT_SYMBOL(__rt_rwlock_init_annotated); +#endif + /* * rw_semaphores */ @@ -347,7 +449,7 @@ int fastcall rt_down_write_trylock(struc int fastcall rt_down_write_trylock(struct rw_semaphore *rwsem) { - int ret = rt_mutex_trylock(&rwsem->lock); + int ret = __LOCK_STAT_RT_MUTEX_TRYLOCK(&rwsem->lock); if (ret) rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); @@ -358,14 +460,14 @@ void fastcall rt_down_write(struct rw_se void fastcall rt_down_write(struct rw_semaphore *rwsem) { rwsem_acquire(&rwsem->dep_map, 0, 0, _RET_IP_); - rt_mutex_lock(&rwsem->lock); + __LOCK_STAT_RT_MUTEX_LOCK(&rwsem->lock); } EXPORT_SYMBOL(rt_down_write); void fastcall rt_down_write_nested(struct rw_semaphore *rwsem, int subclass) { rwsem_acquire(&rwsem->dep_map, subclass, 0, _RET_IP_); - rt_mutex_lock(&rwsem->lock); + __LOCK_STAT_RT_MUTEX_LOCK(&rwsem->lock); } EXPORT_SYMBOL(rt_down_write_nested); @@ -386,7 +488,7 @@ int fastcall rt_down_read_trylock(struct } spin_unlock_irqrestore(&rwsem->lock.wait_lock, flags); - ret = rt_mutex_trylock(&rwsem->lock); + ret = __LOCK_STAT_RT_MUTEX_TRYLOCK(&rwsem->lock); if (ret) rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); return ret; @@ -410,7 +512,7 @@ static void __rt_down_read(struct rw_sem return; } spin_unlock_irqrestore(&rwsem->lock.wait_lock, flags); - rt_mutex_lock(&rwsem->lock); + __LOCK_STAT_RT_MUTEX_LOCK(&rwsem->lock); } void fastcall rt_down_read(struct rw_semaphore *rwsem) @@ -445,14 +547,15 @@ void fastcall rt_down_read_non_owner(str return; } spin_unlock_irqrestore(&rwsem->lock.wait_lock, flags); - rt_mutex_lock(&rwsem->lock); + __LOCK_STAT_RT_MUTEX_LOCK(&rwsem->lock); } EXPORT_SYMBOL(rt_down_read_non_owner); #endif void fastcall __rt_rwsem_init(struct rw_semaphore *rwsem, char *name, - struct lock_class_key *key) + struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) { #ifdef CONFIG_DEBUG_LOCK_ALLOC /* @@ -461,7 +564,7 @@ void fastcall __rt_rwsem_init(struct rw_ debug_check_no_locks_freed((void *)rwsem, sizeof(*rwsem)); lockdep_init_map(&rwsem->dep_map, name, key, 0); #endif - __rt_mutex_init(&rwsem->lock, name); + __rt_mutex_init(&rwsem->lock, name __COMMA_LOCK_STAT_NOTE_VARS); rwsem->read_depth = 0; } EXPORT_SYMBOL(__rt_rwsem_init); @@ -490,7 +593,7 @@ void fastcall rt_down(struct semaphore * void fastcall rt_down(struct semaphore *sem) { - rt_mutex_lock(&sem->lock); + __LOCK_STAT_RT_MUTEX_LOCK(&sem->lock); __down_complete(sem); } EXPORT_SYMBOL(rt_down); @@ -499,7 +602,7 @@ int fastcall rt_down_interruptible(struc { int ret; - ret = rt_mutex_lock_interruptible(&sem->lock, 0); + ret = __LOCK_STAT_RT_MUTEX_LOCK_INTERRUPTIBLE(&sem->lock, 0); if (ret) return ret; __down_complete(sem); @@ -519,7 +622,7 @@ int fastcall rt_down_trylock(struct sema * embedded mutex internally. It would be quite complex to remove * these transient failures so lets try it the simple way first: */ - if (rt_mutex_trylock(&sem->lock)) { + if (__LOCK_STAT_RT_MUTEX_TRYLOCK(&sem->lock)) { __down_complete(sem); return 0; } @@ -546,26 +649,27 @@ EXPORT_SYMBOL(rt_up); } EXPORT_SYMBOL(rt_up); -void fastcall __sema_init(struct semaphore *sem, int val, - char *name, char *file, int line) +void fastcall __sema_init(struct semaphore *sem, int val, char *name + __COMMA_LOCK_STAT_FN_DECL, char *_file, int _line) { atomic_set(&sem->count, val); switch (val) { case 0: - __rt_mutex_init(&sem->lock, name); - rt_mutex_lock(&sem->lock); + __rt_mutex_init(&sem->lock, name __COMMA_LOCK_STAT_NOTE_VARS); + rt_mutex_lock_with_ip(&sem->lock, + (unsigned long) __builtin_return_address(0)); break; default: - __rt_mutex_init(&sem->lock, name); + __rt_mutex_init(&sem->lock, name __COMMA_LOCK_STAT_NOTE_VARS); break; } } EXPORT_SYMBOL(__sema_init); -void fastcall __init_MUTEX(struct semaphore *sem, char *name, char *file, - int line) +void fastcall __init_MUTEX(struct semaphore *sem, char *name + __COMMA_LOCK_STAT_FN_DECL, char *_file, int _line) { - __sema_init(sem, 1, name, file, line); + __sema_init(sem, 1, name __COMMA_LOCK_STAT_FN_VAR, _file, _line); } EXPORT_SYMBOL(__init_MUTEX); ============================================================ --- kernel/rtmutex.c 5fbb6943266e0a2de638851c887e331999eaa16d +++ kernel/rtmutex.c bc9b7e9163572d0fb80256d6095f443788683a25 @@ -15,6 +15,10 @@ #include #include +#include +#include +#include + #include "rtmutex_common.h" #ifdef CONFIG_DEBUG_RT_MUTEXES @@ -68,6 +72,11 @@ rt_mutex_set_owner(struct rt_mutex *lock lock->owner = (struct task_struct *)val; } +static inline void rt_mutex_clear_owner(struct rt_mutex *lock) +{ + lock->owner = (struct task_struct *) NULL; +} + static inline void clear_rt_mutex_waiters(struct rt_mutex *lock) { lock->owner = (struct task_struct *) @@ -146,6 +155,7 @@ static void __rt_mutex_adjust_prio(struc if (task->prio != prio) rt_mutex_setprio(task, prio); + /* reschedules task if the priority is lower, holds the run queue lock */ } /* @@ -623,12 +633,14 @@ rt_spin_lock_fastlock(struct rt_mutex *l static inline void rt_spin_lock_fastlock(struct rt_mutex *lock, - void fastcall (*slowfn)(struct rt_mutex *lock)) + void fastcall (*slowfn)(struct rt_mutex *lock + __COMMA_LOCK_STAT_IP_DECL) + __COMMA_LOCK_STAT_IP_DECL) { if (likely(rt_mutex_cmpxchg(lock, NULL, current))) rt_mutex_deadlock_account_lock(lock, current); else - slowfn(lock); + slowfn(lock __COMMA_LOCK_STAT_IP); } static inline void @@ -652,7 +664,7 @@ static void fastcall noinline __sched * sleep/wakeup event loops. */ static void fastcall noinline __sched -rt_spin_lock_slowlock(struct rt_mutex *lock) +rt_spin_lock_slowlock(struct rt_mutex *lock __COMMA_LOCK_STAT_IP_DECL) { struct rt_mutex_waiter waiter; unsigned long saved_state, state, flags; @@ -665,9 +677,19 @@ rt_spin_lock_slowlock(struct rt_mutex *l /* Try to acquire the lock again: */ if (try_to_take_rt_mutex(lock)) { +#ifdef CONFIG_LOCK_STAT + lock_stat_note(&lock->lock_stat, rt_mutex_owner(lock), _ip, + LOCK_STAT_NOTE_TYPE_HANDOFF); +#endif spin_unlock_irqrestore(&lock->wait_lock, flags); return; } +#ifdef CONFIG_LOCK_STAT + else { + lock_stat_note(&lock->lock_stat, rt_mutex_owner(lock), _ip, + LOCK_STAT_NOTE_TYPE_CONTENTION); + } +#endif BUG_ON(rt_mutex_owner(lock) == current); @@ -775,22 +797,41 @@ void __lockfunc rt_spin_lock(spinlock_t void __lockfunc rt_spin_lock(spinlock_t *lock) { - rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock + __COMMA_LOCK_STAT_RET_IP); spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); } EXPORT_SYMBOL(rt_spin_lock); void __lockfunc __rt_spin_lock(struct rt_mutex *lock) { - rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock); + rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock + __COMMA_LOCK_STAT_RET_IP); } EXPORT_SYMBOL(__rt_spin_lock); +#ifdef CONFIG_LOCK_STAT +void __lockfunc rt_spin_lock_with_ip(spinlock_t *lock, unsigned long _ip) +{ + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock, _ip); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); +} +EXPORT_SYMBOL(rt_spin_lock_with_ip); + +void __lockfunc __rt_spin_lock_with_ip(struct rt_mutex *lock, + unsigned long _ip) +{ + rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock, _ip); +} +EXPORT_SYMBOL(__rt_spin_lock_with_ip); +#endif + #ifdef CONFIG_DEBUG_LOCK_ALLOC void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass) { - rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); + rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock + __COMMA_LOCK_STAT_RET_IP); spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); } EXPORT_SYMBOL(rt_spin_lock_nested); @@ -823,10 +864,19 @@ EXPORT_SYMBOL(rt_spin_unlock_wait); } EXPORT_SYMBOL(rt_spin_unlock_wait); +#ifdef CONFIG_LOCK_STAT +#define __RT_MUTEX_TRYLOCK_WITH_IP(a) \ + rt_mutex_trylock_with_ip(a, (unsigned long) __builtin_return_address(0)); +#else +#define __RT_MUTEX_TRYLOCK_WITH_IP(a) \ + rt_mutex_trylock(a); +#endif + int __lockfunc rt_spin_trylock(spinlock_t *lock) { - int ret = rt_mutex_trylock(&lock->lock); + int ret; + ret = __RT_MUTEX_TRYLOCK_WITH_IP(&lock->lock) if (ret) spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); @@ -839,7 +889,8 @@ int __lockfunc rt_spin_trylock_irqsave(s int ret; *flags = 0; - ret = rt_mutex_trylock(&lock->lock); + + ret = __RT_MUTEX_TRYLOCK_WITH_IP(&lock->lock) if (ret) spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); @@ -852,7 +903,11 @@ int _atomic_dec_and_spin_lock(atomic_t * /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ if (atomic_add_unless(atomic, -1, 1)) return 0; +#ifdef CONFIG_LOCK_STAT + rt_spin_lock_with_ip(lock, (unsigned long) __builtin_return_address(0)); +#else rt_spin_lock(lock); +#endif if (atomic_dec_and_test(atomic)) return 1; rt_spin_unlock(lock); @@ -861,7 +916,8 @@ void EXPORT_SYMBOL(_atomic_dec_and_spin_lock); void -__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key) +__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) { #ifdef CONFIG_DEBUG_LOCK_ALLOC /* @@ -870,12 +926,32 @@ __rt_spin_lock_init(spinlock_t *lock, ch debug_check_no_locks_freed((void *)lock, sizeof(*lock)); lockdep_init_map(&lock->dep_map, name, key, 0); #endif - __rt_mutex_init(&lock->lock, name); + __rt_mutex_init(&lock->lock, name __COMMA_LOCK_STAT_NOTE_VARS); } EXPORT_SYMBOL(__rt_spin_lock_init); +#ifdef CONFIG_LOCK_STAT +void +__rt_spin_lock_init_annotated(spinlock_t *lock, char *name, + struct lock_class_key *key, + LOCK_STAT_NOTE_PARAM_DECL, + struct lock_stat *lsobject) +{ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* + * Make sure we are not reinitializing a held lock: + */ + debug_check_no_locks_freed((void *)lock, sizeof(*lock)); + lockdep_init_map(&lock->dep_map, name, key, 0); #endif + __rt_mutex_init_annotated(&lock->lock, name, LOCK_STAT_NOTE_VARS, + lsobject); +} +EXPORT_SYMBOL(__rt_spin_lock_init_annotated); +#endif +#endif + #ifdef CONFIG_PREEMPT_BKL static inline int rt_release_bkl(struct rt_mutex *lock, unsigned long flags) @@ -914,11 +990,15 @@ rt_mutex_slowlock(struct rt_mutex *lock, static int __sched rt_mutex_slowlock(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, - int detect_deadlock) + int detect_deadlock + __COMMA_LOCK_STAT_IP_DECL) { int ret = 0, saved_lock_depth = -1; struct rt_mutex_waiter waiter; unsigned long flags; +#ifdef CONFIG_LOCK_STAT + struct task_struct *owner; +#endif debug_rt_mutex_init_waiter(&waiter); waiter.task = NULL; @@ -928,16 +1008,30 @@ rt_mutex_slowlock(struct rt_mutex *lock, /* Try to acquire the lock again: */ if (try_to_take_rt_mutex(lock)) { +#ifdef CONFIG_LOCK_STAT + lock_stat_note(&lock->lock_stat, rt_mutex_owner(lock), _ip, + LOCK_STAT_NOTE_TYPE_HANDOFF); +#endif spin_unlock_irqrestore(&lock->wait_lock, flags); return 0; } +#ifdef CONFIG_LOCK_STAT + else { + lock_stat_note(&lock->lock_stat, rt_mutex_owner(lock), _ip, + LOCK_STAT_NOTE_TYPE_CONTENTION); + } +#endif /* * We drop the BKL here before we go into the wait loop to avoid a * possible deadlock in the scheduler. */ if (unlikely(current->lock_depth >= 0)) +#ifdef CONFIG_PREEMPT_BKL saved_lock_depth = rt_release_bkl(lock, flags); +#else + saved_lock_depth = rt_release_bkl(lock); +#endif set_current_state(state); @@ -1040,11 +1134,17 @@ static inline int * Slow path try-lock function: */ static inline int -rt_mutex_slowtrylock(struct rt_mutex *lock) +rt_mutex_slowtrylock(struct rt_mutex *lock __COMMA_LOCK_STAT_IP_DECL) { unsigned long flags; int ret = 0; +#ifdef CONFIG_LOCK_STAT + struct task_struct *owner; + int type; +#endif + debug_rt_mutex_init_waiter(&waiter); + spin_lock_irqsave(&lock->wait_lock, flags); if (likely(rt_mutex_owner(lock) != current)) { @@ -1057,8 +1157,17 @@ rt_mutex_slowtrylock(struct rt_mutex *lo * bit unconditionally. Clean this up. */ fixup_rt_mutex_waiters(lock); + } +#ifdef CONFIG_LOCK_STAT + if (ret) + type = LOCK_STAT_NOTE_TYPE_HANDOFF; + else + type = LOCK_STAT_NOTE_TYPE_CONTENTION; + + lock_stat_note(&lock->lock_stat, rt_mutex_owner(lock), _ip, type); +#endif spin_unlock_irqrestore(&lock->wait_lock, flags); return ret; @@ -1103,38 +1212,62 @@ rt_mutex_fastlock(struct rt_mutex *lock, int detect_deadlock, int (*slowfn)(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, - int detect_deadlock)) + int detect_deadlock __COMMA_LOCK_STAT_IP_DECL) + __COMMA_LOCK_STAT_IP_DECL) { if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 0; } else - return slowfn(lock, state, NULL, detect_deadlock); + return slowfn(lock, state, NULL, detect_deadlock + __COMMA_LOCK_STAT_RET_IP); } +#ifdef CONFIG_LOCK_STAT static inline int +rt_mutex_fastlock_with_ip(struct rt_mutex *lock, int state, + int detect_deadlock, + int (*slowfn)(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, + int detect_deadlock, unsigned long _ip), + unsigned long _ip) +{ + if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { + rt_mutex_deadlock_account_lock(lock, current); + return 0; + } else + return slowfn(lock, state, NULL, detect_deadlock, _ip); +} +#endif + +static inline int rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, int detect_deadlock, int (*slowfn)(struct rt_mutex *lock, int state, struct hrtimer_sleeper *timeout, - int detect_deadlock)) + int detect_deadlock + __COMMA_LOCK_STAT_IP_DECL) + __COMMA_LOCK_STAT_IP_DECL) { if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 0; } else - return slowfn(lock, state, timeout, detect_deadlock); + return slowfn(lock, state, timeout, detect_deadlock + __COMMA_LOCK_STAT_IP); } static inline int rt_mutex_fasttrylock(struct rt_mutex *lock, - int (*slowfn)(struct rt_mutex *lock)) + int (*slowfn)(struct rt_mutex *lock + __COMMA_LOCK_STAT_IP_DECL) + __COMMA_LOCK_STAT_IP_DECL) { if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 1; } - return slowfn(lock); + return slowfn(lock __COMMA_LOCK_STAT_IP); } static inline void @@ -1156,10 +1289,25 @@ void __sched rt_mutex_lock(struct rt_mut { might_sleep(); - rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock); + rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, + rt_mutex_slowlock + __COMMA_LOCK_STAT_RET_IP); } EXPORT_SYMBOL_GPL(rt_mutex_lock); +#ifdef CONFIG_LOCK_STAT +void __sched rt_mutex_lock_with_ip(struct rt_mutex *lock, unsigned long ip) +{ + might_sleep(); + + rt_mutex_fastlock_with_ip(lock, TASK_UNINTERRUPTIBLE, 0, + rt_mutex_slowlock, + ip); +} +EXPORT_SYMBOL_GPL(rt_mutex_lock_with_ip); +#endif + + /** * rt_mutex_lock_interruptible - lock a rt_mutex interruptible * @@ -1175,11 +1323,23 @@ int __sched rt_mutex_lock_interruptible( int detect_deadlock) { might_sleep(); + return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, + detect_deadlock, rt_mutex_slowlock + __COMMA_LOCK_STAT_RET_IP); +} +EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); +#ifdef CONFIG_LOCK_STAT +int __sched rt_mutex_lock_interruptible_with_ip(struct rt_mutex *lock, + int detect_deadlock, + unsigned long ip) +{ + might_sleep(); return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, - detect_deadlock, rt_mutex_slowlock); + detect_deadlock, rt_mutex_slowlock, ip); } -EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); +EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible_with_ip); +#endif /** * rt_mutex_lock_interruptible_ktime - lock a rt_mutex interruptible @@ -1203,7 +1363,8 @@ rt_mutex_timed_lock(struct rt_mutex *loc might_sleep(); return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, - detect_deadlock, rt_mutex_slowlock); + detect_deadlock, rt_mutex_slowlock + __COMMA_LOCK_STAT_RET_IP); } EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); @@ -1216,7 +1377,8 @@ int __sched rt_mutex_trylock(struct rt_m */ int __sched rt_mutex_trylock(struct rt_mutex *lock) { - return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock); + return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock + __COMMA_LOCK_STAT_RET_IP); } EXPORT_SYMBOL_GPL(rt_mutex_trylock); @@ -1231,6 +1393,14 @@ EXPORT_SYMBOL_GPL(rt_mutex_unlock); } EXPORT_SYMBOL_GPL(rt_mutex_unlock); +#ifdef CONFIG_LOCK_STAT +int __sched rt_mutex_trylock_with_ip(struct rt_mutex *lock, unsigned long _ip) +{ + return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock, _ip); +} +EXPORT_SYMBOL_GPL(rt_mutex_trylock_with_ip); +#endif + /*** * rt_mutex_destroy - mark a mutex unusable * @lock: the mutex to be destroyed @@ -1258,16 +1428,50 @@ EXPORT_SYMBOL_GPL(rt_mutex_destroy); * * Initializing of a locked rt lock is not allowed */ -void __rt_mutex_init(struct rt_mutex *lock, const char *name) +void __rt_mutex_init(struct rt_mutex *lock, const char *name + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) { - lock->owner = NULL; - spin_lock_init(&lock->wait_lock); + rt_mutex_clear_owner(lock); + spin_lock_init(&lock->wait_lock); /* raw spinlock here */ plist_head_init(&lock->wait_list, &lock->wait_lock); debug_rt_mutex_init(lock, name); +#ifdef CONFIG_LOCK_STAT + lock->lock_stat = NULL; + lock_stat_scoped_attach(&lock->lock_stat, LOCK_STAT_NOTE_VARS); +#endif } EXPORT_SYMBOL_GPL(__rt_mutex_init); +/* + * Annotated version of lock_stat + */ +#ifdef CONFIG_LOCK_STAT +void __rt_mutex_init_annotated(struct rt_mutex *lock, const char *name, + LOCK_STAT_NOTE_PARAM_DECL, + struct lock_stat *lsobject) +{ + rt_mutex_clear_owner(lock); + spin_lock_init(&lock->wait_lock); /* raw spinlock here */ + plist_head_init(&lock->wait_list, &lock->wait_lock); + + debug_rt_mutex_init(lock, name); + + BUG_ON(!lsobject); + lock->lock_stat = lsobject; + + if(!lock_stat_is_initialized(&lock->lock_stat)) + { + ksym_strcpy(lock->lock_stat->function, _function); + lock->lock_stat->file = (char *) _file; + lock->lock_stat->line = _line; + } + + atomic_inc(&lock->lock_stat->ninlined); +} +#endif +EXPORT_SYMBOL_GPL(__rt_mutex_init_annotated); + /** * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a * proxy owner @@ -1281,7 +1485,7 @@ void rt_mutex_init_proxy_locked(struct r void rt_mutex_init_proxy_locked(struct rt_mutex *lock, struct task_struct *proxy_owner) { - __rt_mutex_init(lock, NULL); + __rt_mutex_init(lock, NULL __COMMA_LOCK_STAT_NOTE); debug_rt_mutex_proxy_lock(lock, proxy_owner); rt_mutex_set_owner(lock, proxy_owner, 0); rt_mutex_deadlock_account_lock(lock, proxy_owner); ============================================================ --- kernel/sched.c cf65e71d0f9406fd8b7c0d142504acbe58858f93 +++ kernel/sched.c 418dd6273f492b996d5665c78e48dcb88f1a9b4a @@ -350,6 +350,11 @@ static inline int cpu_of(struct rq *rq) # define _finish_arch_switch(prev) do { } while (0) #endif +struct task_struct *get_task_on_rq(int cpu) +{ + return per_cpu(runqueues, (cpu)).curr; +} + #ifndef __ARCH_WANT_UNLOCKED_CTXSW static inline int task_running(struct rq *rq, struct task_struct *p) { ============================================================ --- net/tipc/node.c f60bf9ad7d6c43937acab556ff324cbb10ea46fb +++ net/tipc/node.c 9944ff3bd01fba317ee36a9cba2365e3b2ef3a66 @@ -76,7 +76,7 @@ struct node *tipc_node_create(u32 addr) } n_ptr->addr = addr; - spin_lock_init(&n_ptr->lock); + spin_lock_init(&n_ptr->lock); INIT_LIST_HEAD(&n_ptr->nsub); n_ptr->owner = c_ptr; tipc_cltr_attach_node(c_ptr, n_ptr);