ea81174789
Mike reported that commit 7d1a9417 ("x86: Use generic idle loop")
regressed several workloads and caused excessive reschedule
interrupts.
The patch in question failed to notice that the x86 code had an
inverted sense of the polling state versus the new generic code (x86:
default polling, generic: default !polling).
Fix the two prominent x86 mwait based idle drivers and introduce a few
new generic polling helpers (fixing the wrong smp_mb__after_clear_bit
usage).
Also switch the idle routines to using tif_need_resched() which is an
immediate TIF_NEED_RESCHED test as opposed to need_resched which will
end up being slightly different.
Reported-by: Mike Galbraith <bitbucket@online.de>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Cc: lenb@kernel.org
Cc: tglx@linutronix.de
Link: http://lkml.kernel.org/n/tip-nc03imb0etuefmzybzj7sprf@git.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
166 lines
4.3 KiB
C
166 lines
4.3 KiB
C
/* thread_info.h: common low-level thread information accessors
|
|
*
|
|
* Copyright (C) 2002 David Howells (dhowells@redhat.com)
|
|
* - Incorporating suggestions made by Linus Torvalds
|
|
*/
|
|
|
|
#ifndef _LINUX_THREAD_INFO_H
|
|
#define _LINUX_THREAD_INFO_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/bug.h>
|
|
|
|
struct timespec;
|
|
struct compat_timespec;
|
|
|
|
/*
|
|
* System call restart block.
|
|
*/
|
|
struct restart_block {
|
|
long (*fn)(struct restart_block *);
|
|
union {
|
|
/* For futex_wait and futex_wait_requeue_pi */
|
|
struct {
|
|
u32 __user *uaddr;
|
|
u32 val;
|
|
u32 flags;
|
|
u32 bitset;
|
|
u64 time;
|
|
u32 __user *uaddr2;
|
|
} futex;
|
|
/* For nanosleep */
|
|
struct {
|
|
clockid_t clockid;
|
|
struct timespec __user *rmtp;
|
|
#ifdef CONFIG_COMPAT
|
|
struct compat_timespec __user *compat_rmtp;
|
|
#endif
|
|
u64 expires;
|
|
} nanosleep;
|
|
/* For poll */
|
|
struct {
|
|
struct pollfd __user *ufds;
|
|
int nfds;
|
|
int has_timeout;
|
|
unsigned long tv_sec;
|
|
unsigned long tv_nsec;
|
|
} poll;
|
|
};
|
|
};
|
|
|
|
extern long do_no_restart_syscall(struct restart_block *parm);
|
|
|
|
#include <linux/bitops.h>
|
|
#include <asm/thread_info.h>
|
|
|
|
#ifdef __KERNEL__
|
|
|
|
#ifdef CONFIG_DEBUG_STACK_USAGE
|
|
# define THREADINFO_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO)
|
|
#else
|
|
# define THREADINFO_GFP (GFP_KERNEL | __GFP_NOTRACK)
|
|
#endif
|
|
|
|
#define THREADINFO_GFP_ACCOUNTED (THREADINFO_GFP | __GFP_KMEMCG)
|
|
|
|
/*
|
|
* flag set/clear/test wrappers
|
|
* - pass TIF_xxxx constants to these functions
|
|
*/
|
|
|
|
static inline void set_ti_thread_flag(struct thread_info *ti, int flag)
|
|
{
|
|
set_bit(flag, (unsigned long *)&ti->flags);
|
|
}
|
|
|
|
static inline void clear_ti_thread_flag(struct thread_info *ti, int flag)
|
|
{
|
|
clear_bit(flag, (unsigned long *)&ti->flags);
|
|
}
|
|
|
|
static inline int test_and_set_ti_thread_flag(struct thread_info *ti, int flag)
|
|
{
|
|
return test_and_set_bit(flag, (unsigned long *)&ti->flags);
|
|
}
|
|
|
|
static inline int test_and_clear_ti_thread_flag(struct thread_info *ti, int flag)
|
|
{
|
|
return test_and_clear_bit(flag, (unsigned long *)&ti->flags);
|
|
}
|
|
|
|
static inline int test_ti_thread_flag(struct thread_info *ti, int flag)
|
|
{
|
|
return test_bit(flag, (unsigned long *)&ti->flags);
|
|
}
|
|
|
|
#define set_thread_flag(flag) \
|
|
set_ti_thread_flag(current_thread_info(), flag)
|
|
#define clear_thread_flag(flag) \
|
|
clear_ti_thread_flag(current_thread_info(), flag)
|
|
#define test_and_set_thread_flag(flag) \
|
|
test_and_set_ti_thread_flag(current_thread_info(), flag)
|
|
#define test_and_clear_thread_flag(flag) \
|
|
test_and_clear_ti_thread_flag(current_thread_info(), flag)
|
|
#define test_thread_flag(flag) \
|
|
test_ti_thread_flag(current_thread_info(), flag)
|
|
|
|
static inline __deprecated void set_need_resched(void)
|
|
{
|
|
/*
|
|
* Use of this function in deprecated.
|
|
*
|
|
* As of this writing there are only a few users in the DRM tree left
|
|
* all of which are wrong and can be removed without causing too much
|
|
* grief.
|
|
*
|
|
* The DRM people are aware and are working on removing the last few
|
|
* instances.
|
|
*/
|
|
}
|
|
|
|
#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED)
|
|
|
|
#if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK
|
|
/*
|
|
* An arch can define its own version of set_restore_sigmask() to get the
|
|
* job done however works, with or without TIF_RESTORE_SIGMASK.
|
|
*/
|
|
#define HAVE_SET_RESTORE_SIGMASK 1
|
|
|
|
/**
|
|
* set_restore_sigmask() - make sure saved_sigmask processing gets done
|
|
*
|
|
* This sets TIF_RESTORE_SIGMASK and ensures that the arch signal code
|
|
* will run before returning to user mode, to process the flag. For
|
|
* all callers, TIF_SIGPENDING is already set or it's no harm to set
|
|
* it. TIF_RESTORE_SIGMASK need not be in the set of bits that the
|
|
* arch code will notice on return to user mode, in case those bits
|
|
* are scarce. We set TIF_SIGPENDING here to ensure that the arch
|
|
* signal code always gets run when TIF_RESTORE_SIGMASK is set.
|
|
*/
|
|
static inline void set_restore_sigmask(void)
|
|
{
|
|
set_thread_flag(TIF_RESTORE_SIGMASK);
|
|
WARN_ON(!test_thread_flag(TIF_SIGPENDING));
|
|
}
|
|
static inline void clear_restore_sigmask(void)
|
|
{
|
|
clear_thread_flag(TIF_RESTORE_SIGMASK);
|
|
}
|
|
static inline bool test_restore_sigmask(void)
|
|
{
|
|
return test_thread_flag(TIF_RESTORE_SIGMASK);
|
|
}
|
|
static inline bool test_and_clear_restore_sigmask(void)
|
|
{
|
|
return test_and_clear_thread_flag(TIF_RESTORE_SIGMASK);
|
|
}
|
|
#endif /* TIF_RESTORE_SIGMASK && !HAVE_SET_RESTORE_SIGMASK */
|
|
|
|
#ifndef HAVE_SET_RESTORE_SIGMASK
|
|
#error "no set_restore_sigmask() provided and default one won't work"
|
|
#endif
|
|
|
|
#endif /* __KERNEL__ */
|
|
|
|
#endif /* _LINUX_THREAD_INFO_H */
|