0d66bf6d35
Change mutex contention behaviour such that it will sometimes busy wait on acquisition - moving its behaviour closer to that of spinlocks. This concept got ported to mainline from the -rt tree, where it was originally implemented for rtmutexes by Steven Rostedt, based on work by Gregory Haskins. Testing with Ingo's test-mutex application (http://lkml.org/lkml/2006/1/8/50) gave a 345% boost for VFS scalability on my testbox: # ./test-mutex-shm V 16 10 | grep "^avg ops" avg ops/sec: 296604 # ./test-mutex-shm V 16 10 | grep "^avg ops" avg ops/sec: 85870 The key criteria for the busy wait is that the lock owner has to be running on a (different) cpu. The idea is that as long as the owner is running, there is a fair chance it'll release the lock soon, and thus we'll be better off spinning instead of blocking/scheduling. Since regular mutexes (as opposed to rtmutexes) do not atomically track the owner, we add the owner in a non-atomic fashion and deal with the races in the slowpath. Furthermore, to ease the testing of the performance impact of this new code, there is means to disable this behaviour runtime (without having to reboot the system), when scheduler debugging is enabled (CONFIG_SCHED_DEBUG=y), by issuing the following command: # echo NO_OWNER_SPIN > /debug/sched_features This command re-enables spinning again (this is also the default): # echo OWNER_SPIN > /debug/sched_features Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Signed-off-by: Ingo Molnar <mingo@elte.hu>
109 lines
2.9 KiB
C
109 lines
2.9 KiB
C
/*
|
|
* kernel/mutex-debug.c
|
|
*
|
|
* Debugging code for mutexes
|
|
*
|
|
* Started by Ingo Molnar:
|
|
*
|
|
* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
|
|
*
|
|
* lock debugging, locking tree, deadlock detection started by:
|
|
*
|
|
* Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey
|
|
* Released under the General Public License (GPL).
|
|
*/
|
|
#include <linux/mutex.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/module.h>
|
|
#include <linux/poison.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/kallsyms.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/debug_locks.h>
|
|
|
|
#include "mutex-debug.h"
|
|
|
|
/*
|
|
* Must be called with lock->wait_lock held.
|
|
*/
|
|
void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)
|
|
{
|
|
memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter));
|
|
waiter->magic = waiter;
|
|
INIT_LIST_HEAD(&waiter->list);
|
|
}
|
|
|
|
void debug_mutex_wake_waiter(struct mutex *lock, struct mutex_waiter *waiter)
|
|
{
|
|
SMP_DEBUG_LOCKS_WARN_ON(!spin_is_locked(&lock->wait_lock));
|
|
DEBUG_LOCKS_WARN_ON(list_empty(&lock->wait_list));
|
|
DEBUG_LOCKS_WARN_ON(waiter->magic != waiter);
|
|
DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
|
|
}
|
|
|
|
void debug_mutex_free_waiter(struct mutex_waiter *waiter)
|
|
{
|
|
DEBUG_LOCKS_WARN_ON(!list_empty(&waiter->list));
|
|
memset(waiter, MUTEX_DEBUG_FREE, sizeof(*waiter));
|
|
}
|
|
|
|
void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
|
|
struct thread_info *ti)
|
|
{
|
|
SMP_DEBUG_LOCKS_WARN_ON(!spin_is_locked(&lock->wait_lock));
|
|
|
|
/* Mark the current thread as blocked on the lock: */
|
|
ti->task->blocked_on = waiter;
|
|
}
|
|
|
|
void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
|
|
struct thread_info *ti)
|
|
{
|
|
DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
|
|
DEBUG_LOCKS_WARN_ON(waiter->task != ti->task);
|
|
DEBUG_LOCKS_WARN_ON(ti->task->blocked_on != waiter);
|
|
ti->task->blocked_on = NULL;
|
|
|
|
list_del_init(&waiter->list);
|
|
waiter->task = NULL;
|
|
}
|
|
|
|
void debug_mutex_unlock(struct mutex *lock)
|
|
{
|
|
if (unlikely(!debug_locks))
|
|
return;
|
|
|
|
DEBUG_LOCKS_WARN_ON(lock->magic != lock);
|
|
DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
|
|
DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
|
|
mutex_clear_owner(lock);
|
|
}
|
|
|
|
void debug_mutex_init(struct mutex *lock, const char *name,
|
|
struct lock_class_key *key)
|
|
{
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
/*
|
|
* Make sure we are not reinitializing a held lock:
|
|
*/
|
|
debug_check_no_locks_freed((void *)lock, sizeof(*lock));
|
|
lockdep_init_map(&lock->dep_map, name, key, 0);
|
|
#endif
|
|
lock->magic = lock;
|
|
}
|
|
|
|
/***
|
|
* mutex_destroy - mark a mutex unusable
|
|
* @lock: the mutex to be destroyed
|
|
*
|
|
* This function marks the mutex uninitialized, and any subsequent
|
|
* use of the mutex is forbidden. The mutex must not be locked when
|
|
* this function is called.
|
|
*/
|
|
void mutex_destroy(struct mutex *lock)
|
|
{
|
|
DEBUG_LOCKS_WARN_ON(mutex_is_locked(lock));
|
|
lock->magic = NULL;
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(mutex_destroy);
|