[MIPS] Type proof reimplementation of cmpxchg.
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
This commit is contained in:
Ralf Baechle
parent
f6a9e6dec5
commit
fef74705ea
3 changed files with 109 additions and 260 deletions
107
include/asm-mips/cmpxchg.h
Normal file
107
include/asm-mips/cmpxchg.h
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
/*
|
||||
* This file is subject to the terms and conditions of the GNU General Public
|
||||
* License. See the file "COPYING" in the main directory of this archive
|
||||
* for more details.
|
||||
*
|
||||
* Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
|
||||
*/
|
||||
#ifndef __ASM_CMPXCHG_H
|
||||
#define __ASM_CMPXCHG_H
|
||||
|
||||
#include <linux/irqflags.h>
|
||||
|
||||
#define __HAVE_ARCH_CMPXCHG 1
|
||||
|
||||
#define __cmpxchg_asm(ld, st, m, old, new) \
|
||||
({ \
|
||||
__typeof(*(m)) __ret; \
|
||||
\
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) { \
|
||||
__asm__ __volatile__( \
|
||||
" .set push \n" \
|
||||
" .set noat \n" \
|
||||
" .set mips3 \n" \
|
||||
"1: " ld " %0, %2 # __cmpxchg_asm \n" \
|
||||
" bne %0, %z3, 2f \n" \
|
||||
" .set mips0 \n" \
|
||||
" move $1, %z4 \n" \
|
||||
" .set mips3 \n" \
|
||||
" " st " $1, %1 \n" \
|
||||
" beqzl $1, 1b \n" \
|
||||
"2: \n" \
|
||||
" .set pop \n" \
|
||||
: "=&r" (__ret), "=R" (*m) \
|
||||
: "R" (*m), "Jr" (old), "Jr" (new) \
|
||||
: "memory"); \
|
||||
} else if (cpu_has_llsc) { \
|
||||
__asm__ __volatile__( \
|
||||
" .set push \n" \
|
||||
" .set noat \n" \
|
||||
" .set mips3 \n" \
|
||||
"1: " ld " %0, %2 # __cmpxchg_asm \n" \
|
||||
" bne %0, %z3, 2f \n" \
|
||||
" .set mips0 \n" \
|
||||
" move $1, %z4 \n" \
|
||||
" .set mips3 \n" \
|
||||
" " st " $1, %1 \n" \
|
||||
" beqz $1, 3f \n" \
|
||||
"2: \n" \
|
||||
" .subsection 2 \n" \
|
||||
"3: b 1b \n" \
|
||||
" .previous \n" \
|
||||
" .set pop \n" \
|
||||
: "=&r" (__ret), "=R" (*m) \
|
||||
: "R" (*m), "Jr" (old), "Jr" (new) \
|
||||
: "memory"); \
|
||||
} else { \
|
||||
unsigned long __flags; \
|
||||
\
|
||||
raw_local_irq_save(__flags); \
|
||||
__ret = *m; \
|
||||
if (__ret == old) \
|
||||
*m = new; \
|
||||
raw_local_irq_restore(__flags); \
|
||||
} \
|
||||
\
|
||||
__ret; \
|
||||
})
|
||||
|
||||
/*
|
||||
* This function doesn't exist, so you'll get a linker error
|
||||
* if something tries to do an invalid cmpxchg().
|
||||
*/
|
||||
extern void __cmpxchg_called_with_bad_pointer(void);
|
||||
|
||||
#define __cmpxchg(ptr,old,new,barrier) \
|
||||
({ \
|
||||
__typeof__(ptr) __ptr = (ptr); \
|
||||
__typeof__(*(ptr)) __old = (old); \
|
||||
__typeof__(*(ptr)) __new = (new); \
|
||||
__typeof__(*(ptr)) __res = 0; \
|
||||
\
|
||||
barrier; \
|
||||
\
|
||||
switch (sizeof(*(__ptr))) { \
|
||||
case 4: \
|
||||
__res = __cmpxchg_asm("ll", "sc", __ptr, __old, __new); \
|
||||
break; \
|
||||
case 8: \
|
||||
if (sizeof(long) == 8) { \
|
||||
__res = __cmpxchg_asm("lld", "scd", __ptr, \
|
||||
__old, __new); \
|
||||
break; \
|
||||
} \
|
||||
default: \
|
||||
__cmpxchg_called_with_bad_pointer(); \
|
||||
break; \
|
||||
} \
|
||||
\
|
||||
barrier; \
|
||||
\
|
||||
__res; \
|
||||
})
|
||||
|
||||
#define cmpxchg(ptr, old, new) __cmpxchg(ptr, old, new, smp_llsc_mb())
|
||||
#define cmpxchg_local(ptr, old, new) __cmpxchg(ptr, old, new,)
|
||||
|
||||
#endif /* __ASM_CMPXCHG_H */
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
#include <linux/percpu.h>
|
||||
#include <linux/bitops.h>
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/cmpxchg.h>
|
||||
#include <asm/war.h>
|
||||
|
||||
typedef struct
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
#include <asm/addrspace.h>
|
||||
#include <asm/barrier.h>
|
||||
#include <asm/cmpxchg.h>
|
||||
#include <asm/cpu-features.h>
|
||||
#include <asm/dsp.h>
|
||||
#include <asm/war.h>
|
||||
|
|
@ -194,266 +195,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz
|
|||
|
||||
#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
|
||||
|
||||
#define __HAVE_ARCH_CMPXCHG 1
|
||||
|
||||
static inline unsigned long __cmpxchg_u32(volatile int * m, unsigned long old,
|
||||
unsigned long new)
|
||||
{
|
||||
__u32 retval;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: ll %0, %2 # __cmpxchg_u32 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" .set mips0 \n"
|
||||
" move $1, %z4 \n"
|
||||
" .set mips3 \n"
|
||||
" sc $1, %1 \n"
|
||||
" beqzl $1, 1b \n"
|
||||
"2: \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: ll %0, %2 # __cmpxchg_u32 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" .set mips0 \n"
|
||||
" move $1, %z4 \n"
|
||||
" .set mips3 \n"
|
||||
" sc $1, %1 \n"
|
||||
" beqz $1, 3f \n"
|
||||
"2: \n"
|
||||
" .subsection 2 \n"
|
||||
"3: b 1b \n"
|
||||
" .previous \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else {
|
||||
unsigned long flags;
|
||||
|
||||
raw_local_irq_save(flags);
|
||||
retval = *m;
|
||||
if (retval == old)
|
||||
*m = new;
|
||||
raw_local_irq_restore(flags); /* implies memory barrier */
|
||||
}
|
||||
|
||||
smp_llsc_mb();
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
static inline unsigned long __cmpxchg_u32_local(volatile int * m,
|
||||
unsigned long old, unsigned long new)
|
||||
{
|
||||
__u32 retval;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: ll %0, %2 # __cmpxchg_u32 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" .set mips0 \n"
|
||||
" move $1, %z4 \n"
|
||||
" .set mips3 \n"
|
||||
" sc $1, %1 \n"
|
||||
" beqzl $1, 1b \n"
|
||||
"2: \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: ll %0, %2 # __cmpxchg_u32 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" .set mips0 \n"
|
||||
" move $1, %z4 \n"
|
||||
" .set mips3 \n"
|
||||
" sc $1, %1 \n"
|
||||
" beqz $1, 1b \n"
|
||||
"2: \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else {
|
||||
unsigned long flags;
|
||||
|
||||
local_irq_save(flags);
|
||||
retval = *m;
|
||||
if (retval == old)
|
||||
*m = new;
|
||||
local_irq_restore(flags); /* implies memory barrier */
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_64BIT
|
||||
static inline unsigned long __cmpxchg_u64(volatile int * m, unsigned long old,
|
||||
unsigned long new)
|
||||
{
|
||||
__u64 retval;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: lld %0, %2 # __cmpxchg_u64 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" move $1, %z4 \n"
|
||||
" scd $1, %1 \n"
|
||||
" beqzl $1, 1b \n"
|
||||
"2: \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: lld %0, %2 # __cmpxchg_u64 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" move $1, %z4 \n"
|
||||
" scd $1, %1 \n"
|
||||
" beqz $1, 3f \n"
|
||||
"2: \n"
|
||||
" .subsection 2 \n"
|
||||
"3: b 1b \n"
|
||||
" .previous \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else {
|
||||
unsigned long flags;
|
||||
|
||||
raw_local_irq_save(flags);
|
||||
retval = *m;
|
||||
if (retval == old)
|
||||
*m = new;
|
||||
raw_local_irq_restore(flags); /* implies memory barrier */
|
||||
}
|
||||
|
||||
smp_llsc_mb();
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
static inline unsigned long __cmpxchg_u64_local(volatile int * m,
|
||||
unsigned long old, unsigned long new)
|
||||
{
|
||||
__u64 retval;
|
||||
|
||||
if (cpu_has_llsc && R10000_LLSC_WAR) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: lld %0, %2 # __cmpxchg_u64 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" move $1, %z4 \n"
|
||||
" scd $1, %1 \n"
|
||||
" beqzl $1, 1b \n"
|
||||
"2: \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else if (cpu_has_llsc) {
|
||||
__asm__ __volatile__(
|
||||
" .set push \n"
|
||||
" .set noat \n"
|
||||
" .set mips3 \n"
|
||||
"1: lld %0, %2 # __cmpxchg_u64 \n"
|
||||
" bne %0, %z3, 2f \n"
|
||||
" move $1, %z4 \n"
|
||||
" scd $1, %1 \n"
|
||||
" beqz $1, 1b \n"
|
||||
"2: \n"
|
||||
" .set pop \n"
|
||||
: "=&r" (retval), "=R" (*m)
|
||||
: "R" (*m), "Jr" (old), "Jr" (new)
|
||||
: "memory");
|
||||
} else {
|
||||
unsigned long flags;
|
||||
|
||||
local_irq_save(flags);
|
||||
retval = *m;
|
||||
if (retval == old)
|
||||
*m = new;
|
||||
local_irq_restore(flags); /* implies memory barrier */
|
||||
}
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
#else
|
||||
extern unsigned long __cmpxchg_u64_unsupported_on_32bit_kernels(
|
||||
volatile int * m, unsigned long old, unsigned long new);
|
||||
#define __cmpxchg_u64 __cmpxchg_u64_unsupported_on_32bit_kernels
|
||||
extern unsigned long __cmpxchg_u64_local_unsupported_on_32bit_kernels(
|
||||
volatile int * m, unsigned long old, unsigned long new);
|
||||
#define __cmpxchg_u64_local __cmpxchg_u64_local_unsupported_on_32bit_kernels
|
||||
#endif
|
||||
|
||||
/* This function doesn't exist, so you'll get a linker error
|
||||
if something tries to do an invalid cmpxchg(). */
|
||||
extern void __cmpxchg_called_with_bad_pointer(void);
|
||||
|
||||
static inline unsigned long __cmpxchg(volatile void * ptr, unsigned long old,
|
||||
unsigned long new, int size)
|
||||
{
|
||||
switch (size) {
|
||||
case 4:
|
||||
return __cmpxchg_u32(ptr, old, new);
|
||||
case 8:
|
||||
return __cmpxchg_u64(ptr, old, new);
|
||||
}
|
||||
__cmpxchg_called_with_bad_pointer();
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline unsigned long __cmpxchg_local(volatile void * ptr,
|
||||
unsigned long old, unsigned long new, int size)
|
||||
{
|
||||
switch (size) {
|
||||
case 4:
|
||||
return __cmpxchg_u32_local(ptr, old, new);
|
||||
case 8:
|
||||
return __cmpxchg_u64_local(ptr, old, new);
|
||||
}
|
||||
__cmpxchg_called_with_bad_pointer();
|
||||
return old;
|
||||
}
|
||||
|
||||
#define cmpxchg(ptr,old,new) \
|
||||
((__typeof__(*(ptr)))__cmpxchg((ptr), \
|
||||
(unsigned long)(old), (unsigned long)(new),sizeof(*(ptr))))
|
||||
|
||||
#define cmpxchg_local(ptr,old,new) \
|
||||
((__typeof__(*(ptr)))__cmpxchg_local((ptr), \
|
||||
(unsigned long)(old), (unsigned long)(new),sizeof(*(ptr))))
|
||||
|
||||
extern void set_handler (unsigned long offset, void *addr, unsigned long len);
|
||||
extern void set_uncached_handler (unsigned long offset, void *addr, unsigned long len);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue