Re: [PATCH v3] parisc: Rewrite light-weight syscall and futex code

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On 12/24/21 19:50, John David Anglin wrote:
> The parisc architecture lacks general hardware support for compare and swap.
> Particularly for userspace, it is difficult to implement software atomic
> support. Page faults in critical regions can cause processes to sleep and
> block the forward progress of other processes.  Thus, it is essential that
> page faults be disabled in critical regions. For performance reasons, we
> also need to disable external interrupts in critical regions.
>
> In order to do this, we need a mechanism to trigger COW breaks outside the
> critical region. Fortunately, parisc has the "stbys,e" instruction. When
> the leftmost byte of a word is addressed, this instruction triggers all
> the exceptions of a normal store but it does not write to memory. Thus,
> we can use it to trigger COW breaks outside the critical region without
> modifying the data that is to be updated atomically.
>
> COW breaks occur randomly.  So even if we have priviously executed a "stbys,e"
> instruction, we still need to disable pagefaults around the critical region.
> If a fault occurs in the critical region, we return -EAGAIN. I had to add
> a wrapper around _arch_futex_atomic_op_inuser() as I found in testing that
> returning -EAGAIN caused problems for some processes even though it is
> listed as a possible return value.
>
> The patch implements the above. The code no longer attempts to sleep with
> interrupts disabled and I haven't seen any stalls with the change.
>
> I have attempted to merge common code and streamline the fast path.  In the
> futex code, we only compute the spinlock address once.
>
> I eliminated some debug code in the original CAS routine that just made the
> flow more complicated.
>
> I don't clip the arguments when called from wide mode. As a result, the LWS
> routines should work when called from 64-bit processes.
>
> I defined TASK_PAGEFAULT_DISABLED offset for use in the lws_pagefault_disable
> and lws_pagefault_enable macros.
>
> Since we now disable interrupts on the gateway page where necessary, it
> might be possible to allow processes to be scheduled when they are on the
> gateway page.
>
> Change has been tested on c8000 and rp3440. It improves glibc build and test
> time by about 10%.
>
> In v2, I removed the lws_atomic_xchg and and lws_atomic_store calls. I
> also removed the bug fixes that were not directly related to this patch.
>
> In v3, I removed the code to force interruptions from
> arch_futex_atomic_op_inuser(). It is always called with page faults
> disabled, so this code had no effect.
>
> Signed-off-by: John David Anglin <dave.anglin@xxxxxxxx>

applied.
Thanks,
Helge


> ---
>
> diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h
> index 9cd4dd6e63ad..b5835325d44b 100644
> --- a/arch/parisc/include/asm/futex.h
> +++ b/arch/parisc/include/asm/futex.h
> @@ -8,39 +8,47 @@
>  #include <asm/errno.h>
>
>  /* The following has to match the LWS code in syscall.S.  We have
> -   sixteen four-word locks. */
> + * 256 four-word locks. We use bits 20-27 of the futex virtual
> + * address for the hash index.
> + */
> +
> +static inline unsigned long _futex_hash_index(unsigned long ua)
> +{
> +	return (ua >> 2) & 0x3fc;
> +}
>
>  static inline void
> -_futex_spin_lock(u32 __user *uaddr)
> +_futex_spin_lock_irqsave(arch_spinlock_t *s, unsigned long *flags)
>  {
> -	extern u32 lws_lock_start[];
> -	long index = ((long)uaddr & 0x7f8) >> 1;
> -	arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index];
> -	preempt_disable();
> +	local_irq_save(*flags);
>  	arch_spin_lock(s);
>  }
>
>  static inline void
> -_futex_spin_unlock(u32 __user *uaddr)
> +_futex_spin_unlock_irqrestore(arch_spinlock_t *s, unsigned long *flags)
>  {
> -	extern u32 lws_lock_start[];
> -	long index = ((long)uaddr & 0x7f8) >> 1;
> -	arch_spinlock_t *s = (arch_spinlock_t *)&lws_lock_start[index];
>  	arch_spin_unlock(s);
> -	preempt_enable();
> +	local_irq_restore(*flags);
>  }
>
>  static inline int
>  arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
>  {
> +	extern u32 lws_lock_start[];
> +	unsigned long ua = (unsigned long)uaddr;
> +	arch_spinlock_t *s;
> +	unsigned long flags;
>  	int oldval, ret;
>  	u32 tmp;
>
> -	ret = -EFAULT;
> +	s = (arch_spinlock_t *)&lws_lock_start[_futex_hash_index(ua)];
> +	_futex_spin_lock_irqsave(s, &flags);
>
> -	_futex_spin_lock(uaddr);
> -	if (unlikely(get_user(oldval, uaddr) != 0))
> +	/* Return -EFAULT if we encounter a page fault or COW break */
> +	if (unlikely(get_user(oldval, uaddr) != 0)) {
> +		ret = -EFAULT;
>  		goto out_pagefault_enable;
> +	}
>
>  	ret = 0;
>  	tmp = oldval;
> @@ -63,13 +71,14 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
>  		break;
>  	default:
>  		ret = -ENOSYS;
> +		goto out_pagefault_enable;
>  	}
>
> -	if (ret == 0 && unlikely(put_user(tmp, uaddr) != 0))
> +	if (unlikely(put_user(tmp, uaddr) != 0))
>  		ret = -EFAULT;
>
>  out_pagefault_enable:
> -	_futex_spin_unlock(uaddr);
> +	_futex_spin_unlock_irqrestore(s, &flags);
>
>  	if (!ret)
>  		*oval = oldval;
> @@ -81,7 +90,11 @@ static inline int
>  futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
>  			      u32 oldval, u32 newval)
>  {
> +	extern u32 lws_lock_start[];
> +	unsigned long ua = (unsigned long)uaddr;
> +	arch_spinlock_t *s;
>  	u32 val;
> +	unsigned long flags;
>
>  	/* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is
>  	 * our gateway page, and causes no end of trouble...
> @@ -94,23 +107,25 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
>
>  	/* HPPA has no cmpxchg in hardware and therefore the
>  	 * best we can do here is use an array of locks. The
> -	 * lock selected is based on a hash of the userspace
> -	 * address. This should scale to a couple of CPUs.
> +	 * lock selected is based on a hash of the virtual
> +	 * address of the futex. This should scale to a couple
> +	 * of CPUs.
>  	 */
>
> -	_futex_spin_lock(uaddr);
> +	s = (arch_spinlock_t *)&lws_lock_start[_futex_hash_index(ua)];
> +	_futex_spin_lock_irqsave(s, &flags);
>  	if (unlikely(get_user(val, uaddr) != 0)) {
> -		_futex_spin_unlock(uaddr);
> +		_futex_spin_unlock_irqrestore(s, &flags);
>  		return -EFAULT;
>  	}
>
>  	if (val == oldval && unlikely(put_user(newval, uaddr) != 0)) {
> -		_futex_spin_unlock(uaddr);
> +		_futex_spin_unlock_irqrestore(s, &flags);
>  		return -EFAULT;
>  	}
>
>  	*uval = val;
> -	_futex_spin_unlock(uaddr);
> +	_futex_spin_unlock_irqrestore(s, &flags);
>
>  	return 0;
>  }
> diff --git a/arch/parisc/kernel/asm-offsets.c b/arch/parisc/kernel/asm-offsets.c
> index 55c1c5189c6a..396aa3b47712 100644
> --- a/arch/parisc/kernel/asm-offsets.c
> +++ b/arch/parisc/kernel/asm-offsets.c
> @@ -37,6 +37,7 @@ int main(void)
>  {
>  	DEFINE(TASK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags));
>  	DEFINE(TASK_STACK, offsetof(struct task_struct, stack));
> +	DEFINE(TASK_PAGEFAULT_DISABLED, offsetof(struct task_struct, pagefault_disabled));
>  	BLANK();
>  	DEFINE(TASK_REGS, offsetof(struct task_struct, thread.regs));
>  	DEFINE(TASK_PT_PSW, offsetof(struct task_struct, thread.regs.gr[ 0]));
> diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S
> index 65c88ca7a7ac..71cb6d17afac 100644
> --- a/arch/parisc/kernel/syscall.S
> +++ b/arch/parisc/kernel/syscall.S
> @@ -50,6 +50,22 @@ registers).
>
>  	.level          PA_ASM_LEVEL
>
> +	.macro	lws_pagefault_disable reg1,reg2
> +	mfctl	%cr30, \reg2
> +	ldo	TASK_PAGEFAULT_DISABLED(\reg2), \reg2
> +	ldw	0(%sr2,\reg2), \reg1
> +	ldo	1(\reg1), \reg1
> +	stw	\reg1, 0(%sr2,\reg2)
> +	.endm
> +
> +	.macro	lws_pagefault_enable reg1,reg2
> +	mfctl	%cr30, \reg2
> +	ldo	TASK_PAGEFAULT_DISABLED(\reg2), \reg2
> +	ldw	0(%sr2,\reg2), \reg1
> +	ldo	-1(\reg1), \reg1
> +	stw	\reg1, 0(%sr2,\reg2)
> +	.endm
> +
>  	.text
>
>  	.import syscall_exit,code
> @@ -490,8 +506,35 @@ lws_start:
>  	/* Jump to lws, lws table pointers already relocated */
>  	be,n	0(%sr2,%r21)
>
> +lws_exit_noerror:
> +	stw,ma	%r20, 0(%sr2,%r20)
> +	ssm	PSW_SM_I, %r0
> +	lws_pagefault_enable	%r1,%r21
> +	b	lws_exit
> +	copy	%r0, %r21
> +
> +lws_wouldblock:
> +	ssm	PSW_SM_I, %r0
> +	lws_pagefault_enable	%r1,%r21
> +	ldo	2(%r0), %r28
> +	b	lws_exit
> +	ldo	-EAGAIN(%r0), %r21
> +
> +lws_pagefault:
> +	stw,ma	%r20, 0(%sr2,%r20)
> +	ssm	PSW_SM_I, %r0
> +	lws_pagefault_enable	%r1,%r21
> +	ldo	3(%r0),%r28
> +	b	lws_exit
> +	ldo	-EAGAIN(%r0),%r21
> +
> +lws_fault:
> +	ldo	1(%r0),%r28
> +	b	lws_exit
> +	ldo	-EFAULT(%r0),%r21
> +
>  lws_exit_nosys:
> -	ldo	-ENOSYS(%r0),%r21		   /* set errno */
> +	ldo	-ENOSYS(%r0),%r21
>  	/* Fall through: Return to userspace */
>
>  lws_exit:
> @@ -518,27 +561,19 @@ lws_exit:
>  		%r28 - Return prev through this register.
>  		%r21 - Kernel error code
>
> -		If debugging is DISabled:
> -
> -		%r21 has the following meanings:
> -
> +		%r21 returns the following error codes:
>  		EAGAIN - CAS is busy, ldcw failed, try again.
>  		EFAULT - Read or write failed.
>
> -		If debugging is enabled:
> -
> -		EDEADLOCK - CAS called recursively.
> -		EAGAIN && r28 == 1 - CAS is busy. Lock contended.
> -		EAGAIN && r28 == 2 - CAS is busy. ldcw failed.
> -		EFAULT - Read or write failed.
> +		If EAGAIN is returned, %r28 indicates the busy reason:
> +		r28 == 1 - CAS is busy. lock contended.
> +		r28 == 2 - CAS is busy. ldcw failed.
> +		r28 == 3 - CAS is busy. page fault.
>
>  		Scratch: r20, r28, r1
>
>  	****************************************************/
>
> -	/* Do not enable LWS debugging */
> -#define ENABLE_LWS_DEBUG 0
> -
>  	/* ELF64 Process entry path */
>  lws_compare_and_swap64:
>  #ifdef CONFIG_64BIT
> @@ -551,59 +586,44 @@ lws_compare_and_swap64:
>  	b,n	lws_exit_nosys
>  #endif
>
> -	/* ELF32 Process entry path */
> +	/* ELF32/ELF64 Process entry path */
>  lws_compare_and_swap32:
>  #ifdef CONFIG_64BIT
> -	/* Clip all the input registers */
> +	/* Wide mode user process? */
> +	bb,<,n  %sp, 31, lws_compare_and_swap
> +
> +	/* Clip all the input registers for 32-bit processes */
>  	depdi	0, 31, 32, %r26
>  	depdi	0, 31, 32, %r25
>  	depdi	0, 31, 32, %r24
>  #endif
>
>  lws_compare_and_swap:
> -	/* Load start of lock table */
> -	ldil	L%lws_lock_start, %r20
> -	ldo	R%lws_lock_start(%r20), %r28
> +	/* Trigger memory reference interruptions without writing to memory */
> +1:	ldw	0(%r26), %r28
> +2:	stbys,e	%r0, 0(%r26)
>
> -	/* Extract eight bits from r26 and hash lock (Bits 3-11) */
> -	extru_safe  %r26, 28, 8, %r20
> +	/* Calculate 8-bit hash index from virtual address */
> +	extru_safe	%r26, 27, 8, %r20
> +
> +	/* Load start of lock table */
> +	ldil	L%lws_lock_start, %r28
> +	ldo	R%lws_lock_start(%r28), %r28
>
> -	/* Find lock to use, the hash is either one of 0 to
> -	   15, multiplied by 16 (keep it 16-byte aligned)
> +	/* Find lock to use, the hash index is one of 0 to
> +	   255, multiplied by 16 (keep it 16-byte aligned)
>  	   and add to the lock table offset. */
>  	shlw	%r20, 4, %r20
>  	add	%r20, %r28, %r20
>
> -# if ENABLE_LWS_DEBUG
> -	/*
> -		DEBUG, check for deadlock!
> -		If the thread register values are the same
> -		then we were the one that locked it last and
> -		this is a recurisve call that will deadlock.
> -		We *must* giveup this call and fail.
> -	*/
> -	ldw	4(%sr2,%r20), %r28			/* Load thread register */
> -	/* WARNING: If cr27 cycles to the same value we have problems */
> -	mfctl	%cr27, %r21				/* Get current thread register */
> -	cmpb,<>,n	%r21, %r28, cas_lock		/* Called recursive? */
> -	b	lws_exit				/* Return error! */
> -	ldo	-EDEADLOCK(%r0), %r21
> -cas_lock:
> -	cmpb,=,n	%r0, %r28, cas_nocontend	/* Is nobody using it? */
> -	ldo	1(%r0), %r28				/* 1st case */
> -	b	lws_exit				/* Contended... */
> -	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
> -cas_nocontend:
> -# endif
> -/* ENABLE_LWS_DEBUG */
> -
> -	/* COW breaks can cause contention on UP systems */
> -	LDCW	0(%sr2,%r20), %r28			/* Try to acquire the lock */
> -	cmpb,<>,n	%r0, %r28, cas_action		/* Did we get it? */
> -cas_wouldblock:
> -	ldo	2(%r0), %r28				/* 2nd case */
> -	b	lws_exit				/* Contended... */
> -	ldo	-EAGAIN(%r0), %r21			/* Spin in userspace */
> +	/* Disable page faults to prevent sleeping in critical region */
> +	lws_pagefault_disable	%r21,%r28
> +	rsm	PSW_SM_I, %r0				/* Disable interrupts */
> +
> +	/* Try to acquire the lock */
> +	LDCW	0(%sr2,%r20), %r28
> +	comclr,<>	%r0, %r28, %r0
> +	b,n	lws_wouldblock
>
>  	/*
>  		prev = *addr;
> @@ -613,59 +633,35 @@ cas_wouldblock:
>  	*/
>
>  	/* NOTES:
> -		This all works becuse intr_do_signal
> +		This all works because intr_do_signal
>  		and schedule both check the return iasq
>  		and see that we are on the kernel page
>  		so this process is never scheduled off
>  		or is ever sent any signal of any sort,
> -		thus it is wholly atomic from usrspaces
> +		thus it is wholly atomic from usrspace's
>  		perspective
>  	*/
> -cas_action:
> -#if defined CONFIG_SMP && ENABLE_LWS_DEBUG
> -	/* DEBUG */
> -	mfctl	%cr27, %r1
> -	stw	%r1, 4(%sr2,%r20)
> -#endif
>  	/* The load and store could fail */
> -1:	ldw	0(%r26), %r28
> +3:	ldw	0(%r26), %r28
>  	sub,<>	%r28, %r25, %r0
> -2:	stw	%r24, 0(%r26)
> -	/* Free lock */
> -	stw,ma	%r20, 0(%sr2,%r20)
> -#if ENABLE_LWS_DEBUG
> -	/* Clear thread register indicator */
> -	stw	%r0, 4(%sr2,%r20)
> -#endif
> -	/* Return to userspace, set no error */
> -	b	lws_exit
> -	copy	%r0, %r21
> +4:	stw	%r24, 0(%r26)
> +	b,n	lws_exit_noerror
>
> -3:
> -	/* Error occurred on load or store */
> -	/* Free lock */
> -	stw,ma	%r20, 0(%sr2,%r20)
> -#if ENABLE_LWS_DEBUG
> -	stw	%r0, 4(%sr2,%r20)
> -#endif
> -	b	lws_exit
> -	ldo	-EFAULT(%r0),%r21	/* set errno */
> -	nop
> -	nop
> -	nop
> -	nop
> +	/* A fault occurred on load or stbys,e store */
> +5:	b,n	lws_fault
> +	ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 5b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 5b-linux_gateway_page)
>
> -	/* Two exception table entries, one for the load,
> -	   the other for the store. Either return -EFAULT.
> -	   Each of the entries must be relocated. */
> -	ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 3b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 3b-linux_gateway_page)
> +	/* A page fault occurred in critical region */
> +6:	b,n	lws_pagefault
> +	ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 6b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 6b-linux_gateway_page)
>
>
>  	/***************************************************
>  		New CAS implementation which uses pointers and variable size
>  		information. The value pointed by old and new MUST NOT change
> -		while performing CAS. The lock only protect the value at %r26.
> +		while performing CAS. The lock only protects the value at %r26.
>
>  		%r26 - Address to examine
>  		%r25 - Pointer to the value to check (old)
> @@ -674,25 +670,32 @@ cas_action:
>  		%r28 - Return non-zero on failure
>  		%r21 - Kernel error code
>
> -		%r21 has the following meanings:
> -
> +		%r21 returns the following error codes:
>  		EAGAIN - CAS is busy, ldcw failed, try again.
>  		EFAULT - Read or write failed.
>
> +		If EAGAIN is returned, %r28 indicates the busy reason:
> +		r28 == 1 - CAS is busy. lock contended.
> +		r28 == 2 - CAS is busy. ldcw failed.
> +		r28 == 3 - CAS is busy. page fault.
> +
>  		Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only)
>
>  	****************************************************/
>
> -	/* ELF32 Process entry path */
>  lws_compare_and_swap_2:
>  #ifdef CONFIG_64BIT
> -	/* Clip the input registers. We don't need to clip %r23 as we
> -	   only use it for word operations */
> +	/* Wide mode user process? */
> +	bb,<,n	%sp, 31, cas2_begin
> +
> +	/* Clip the input registers for 32-bit processes. We don't
> +	   need to clip %r23 as we only use it for word operations */
>  	depdi	0, 31, 32, %r26
>  	depdi	0, 31, 32, %r25
>  	depdi	0, 31, 32, %r24
>  #endif
>
> +cas2_begin:
>  	/* Check the validity of the size pointer */
>  	subi,>>= 3, %r23, %r0
>  	b,n	lws_exit_nosys
> @@ -703,69 +706,76 @@ lws_compare_and_swap_2:
>  	blr	%r29, %r0
>  	nop
>
> -	/* 8bit load */
> -4:	ldb	0(%r25), %r25
> +	/* 8-bit load */
> +1:	ldb	0(%r25), %r25
>  	b	cas2_lock_start
> -5:	ldb	0(%r24), %r24
> +2:	ldb	0(%r24), %r24
>  	nop
>  	nop
>  	nop
>  	nop
>  	nop
>
> -	/* 16bit load */
> -6:	ldh	0(%r25), %r25
> +	/* 16-bit load */
> +3:	ldh	0(%r25), %r25
>  	b	cas2_lock_start
> -7:	ldh	0(%r24), %r24
> +4:	ldh	0(%r24), %r24
>  	nop
>  	nop
>  	nop
>  	nop
>  	nop
>
> -	/* 32bit load */
> -8:	ldw	0(%r25), %r25
> +	/* 32-bit load */
> +5:	ldw	0(%r25), %r25
>  	b	cas2_lock_start
> -9:	ldw	0(%r24), %r24
> +6:	ldw	0(%r24), %r24
>  	nop
>  	nop
>  	nop
>  	nop
>  	nop
>
> -	/* 64bit load */
> +	/* 64-bit load */
>  #ifdef CONFIG_64BIT
> -10:	ldd	0(%r25), %r25
> -11:	ldd	0(%r24), %r24
> +7:	ldd	0(%r25), %r25
> +8:	ldd	0(%r24), %r24
>  #else
>  	/* Load old value into r22/r23 - high/low */
> -10:	ldw	0(%r25), %r22
> -11:	ldw	4(%r25), %r23
> +7:	ldw	0(%r25), %r22
> +8:	ldw	4(%r25), %r23
>  	/* Load new value into fr4 for atomic store later */
> -12:	flddx	0(%r24), %fr4
> +9:	flddx	0(%r24), %fr4
>  #endif
>
>  cas2_lock_start:
> -	/* Load start of lock table */
> -	ldil	L%lws_lock_start, %r20
> -	ldo	R%lws_lock_start(%r20), %r28
> +	/* Trigger memory reference interruptions without writing to memory */
> +	copy	%r26, %r28
> +	depi_safe	0, 31, 2, %r21
> +10:	ldw	0(%r28), %r1
> +11:	stbys,e	%r0, 0(%r28)
>
> -	/* Extract eight bits from r26 and hash lock (Bits 3-11) */
> -	extru_safe  %r26, 28, 8, %r20
> +	/* Calculate 8-bit hash index from virtual address */
> +	extru_safe	%r26, 27, 8, %r20
>
> -	/* Find lock to use, the hash is either one of 0 to
> -	   15, multiplied by 16 (keep it 16-byte aligned)
> +	/* Load start of lock table */
> +	ldil	L%lws_lock_start, %r28
> +	ldo	R%lws_lock_start(%r28), %r28
> +
> +	/* Find lock to use, the hash index is one of 0 to
> +	   255, multiplied by 16 (keep it 16-byte aligned)
>  	   and add to the lock table offset. */
>  	shlw	%r20, 4, %r20
>  	add	%r20, %r28, %r20
>
> -	/* COW breaks can cause contention on UP systems */
> -	LDCW	0(%sr2,%r20), %r28		/* Try to acquire the lock */
> -	cmpb,<>,n	%r0, %r28, cas2_action	/* Did we get it? */
> -cas2_wouldblock:
> -	ldo	2(%r0), %r28			/* 2nd case */
> -	b	lws_exit			/* Contended... */
> -	ldo	-EAGAIN(%r0), %r21		/* Spin in userspace */
> +	/* Disable page faults to prevent sleeping in critical region */
> +	lws_pagefault_disable	%r21,%r28
> +	rsm	PSW_SM_I, %r0			/* Disable interrupts */
> +
> +	/* Try to acquire the lock */
> +	LDCW	0(%sr2,%r20), %r28
> +	comclr,<>	%r0, %r28, %r0
> +	b,n	lws_wouldblock
>
>  	/*
>  		prev = *addr;
> @@ -775,110 +785,102 @@ cas2_wouldblock:
>  	*/
>
>  	/* NOTES:
> -		This all works becuse intr_do_signal
> +		This all works because intr_do_signal
>  		and schedule both check the return iasq
>  		and see that we are on the kernel page
>  		so this process is never scheduled off
>  		or is ever sent any signal of any sort,
> -		thus it is wholly atomic from usrspaces
> +		thus it is wholly atomic from usrspace's
>  		perspective
>  	*/
> -cas2_action:
> +
>  	/* Jump to the correct function */
>  	blr	%r29, %r0
>  	/* Set %r28 as non-zero for now */
>  	ldo	1(%r0),%r28
>
> -	/* 8bit CAS */
> -13:	ldb	0(%r26), %r29
> +	/* 8-bit CAS */
> +12:	ldb	0(%r26), %r29
>  	sub,=	%r29, %r25, %r0
> -	b,n	cas2_end
> -14:	stb	%r24, 0(%r26)
> -	b	cas2_end
> +	b,n	lws_exit_noerror
> +13:	stb	%r24, 0(%r26)
> +	b	lws_exit_noerror
>  	copy	%r0, %r28
>  	nop
>  	nop
>
> -	/* 16bit CAS */
> -15:	ldh	0(%r26), %r29
> +	/* 16-bit CAS */
> +14:	ldh	0(%r26), %r29
>  	sub,=	%r29, %r25, %r0
> -	b,n	cas2_end
> -16:	sth	%r24, 0(%r26)
> -	b	cas2_end
> +	b,n	lws_exit_noerror
> +15:	sth	%r24, 0(%r26)
> +	b	lws_exit_noerror
>  	copy	%r0, %r28
>  	nop
>  	nop
>
> -	/* 32bit CAS */
> -17:	ldw	0(%r26), %r29
> +	/* 32-bit CAS */
> +16:	ldw	0(%r26), %r29
>  	sub,=	%r29, %r25, %r0
> -	b,n	cas2_end
> -18:	stw	%r24, 0(%r26)
> -	b	cas2_end
> +	b,n	lws_exit_noerror
> +17:	stw	%r24, 0(%r26)
> +	b	lws_exit_noerror
>  	copy	%r0, %r28
>  	nop
>  	nop
>
> -	/* 64bit CAS */
> +	/* 64-bit CAS */
>  #ifdef CONFIG_64BIT
> -19:	ldd	0(%r26), %r29
> +18:	ldd	0(%r26), %r29
>  	sub,*=	%r29, %r25, %r0
> -	b,n	cas2_end
> -20:	std	%r24, 0(%r26)
> +	b,n	lws_exit_noerror
> +19:	std	%r24, 0(%r26)
>  	copy	%r0, %r28
>  #else
>  	/* Compare first word */
> -19:	ldw	0(%r26), %r29
> +18:	ldw	0(%r26), %r29
>  	sub,=	%r29, %r22, %r0
> -	b,n	cas2_end
> +	b,n	lws_exit_noerror
>  	/* Compare second word */
> -20:	ldw	4(%r26), %r29
> +19:	ldw	4(%r26), %r29
>  	sub,=	%r29, %r23, %r0
> -	b,n	cas2_end
> +	b,n	lws_exit_noerror
>  	/* Perform the store */
> -21:	fstdx	%fr4, 0(%r26)
> +20:	fstdx	%fr4, 0(%r26)
>  	copy	%r0, %r28
>  #endif
> +	b	lws_exit_noerror
> +	copy	%r0, %r28
>
> -cas2_end:
> -	/* Free lock */
> -	stw,ma	%r20, 0(%sr2,%r20)
> -	/* Return to userspace, set no error */
> -	b	lws_exit
> -	copy	%r0, %r21
> -
> -22:
> -	/* Error occurred on load or store */
> -	/* Free lock */
> -	stw,ma	%r20, 0(%sr2,%r20)
> -	ldo	1(%r0),%r28
> -	b	lws_exit
> -	ldo	-EFAULT(%r0),%r21	/* set errno */
> -	nop
> -	nop
> -	nop
> +	/* A fault occurred on load or stbys,e store */
> +30:	b,n	lws_fault
> +	ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page)
> +#ifndef CONFIG_64BIT
> +	ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page)
> +#endif
>
> -	/* Exception table entries, for the load and store, return EFAULT.
> -	   Each of the entries must be relocated. */
> -	ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 22b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page)
> +
> +	/* A page fault occurred in critical region */
> +31:	b,n	lws_pagefault
> +	ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page)
>  #ifndef CONFIG_64BIT
> -	ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 22b-linux_gateway_page)
> -	ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 22b-linux_gateway_page)
> +	ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page)
>  #endif
>
>  	/* Make sure nothing else is placed on this page */
> @@ -899,7 +901,7 @@ ENTRY(end_linux_gateway_page)
>  ENTRY(lws_table)
>  	LWS_ENTRY(compare_and_swap32)		/* 0 - ELF32 Atomic 32bit CAS */
>  	LWS_ENTRY(compare_and_swap64)		/* 1 - ELF64 Atomic 32bit CAS */
> -	LWS_ENTRY(compare_and_swap_2)		/* 2 - ELF32 Atomic 64bit CAS */
> +	LWS_ENTRY(compare_and_swap_2)		/* 2 - Atomic 64bit CAS */
>  END(lws_table)
>  	/* End of lws table */
>
>





[Index of Archives]     [Linux SoC]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]

  Powered by Linux