ieee754[sd]p_neg workaround

[Atsushi Nemoto <anemo@xxxxxxxxxxxxx>]

I have a long standing patch for FPU emulator to fix a segmentation
fault in pow() library function.

Here is a test program to reproduce it.

main()
{
	union {
		double d;
		struct {
#ifdef __MIPSEB
			unsigned int high, low;
#else
			unsigned int low, high;
#endif
		} i;
	} x, y, z;
        x.i.low = 0x00000000;
        x.i.high = 0xfff00001;
        y.i.low = 0x80000000;
        y.i.high = 0xcff00000;
        z.d = pow(x.d, y.d);
        printf("%x %x\n", z.i.high, z.i.low);
        return 0;
}


If you run this program, you will get segmentation fault (unless your
FPU does not raise Unimplemented exception for NaN operands).  The
segmentation fault is caused by endless recursion in __ieee754_pow().

It looks glibc's pow() assume unary '-' operation for any number
(including NaN) always invert its sign bit.

Here is a revised (just added a few comments) patch.  Please review.
Thank you.

diff -u linux-mips/arch/mips/math-emu/dp_simple.c linux/arch/mips/math-emu/dp_simple.c
--- linux-mips/arch/mips/math-emu/dp_simple.c	2005-01-31 11:05:18.000000000 +0900
+++ linux/arch/mips/math-emu/dp_simple.c	2005-04-20 17:02:02.613112541 +0900
@@ -48,16 +48,22 @@
 	CLEARCX;
 	FLUSHXDP;
 
+	/*
+	 * Invert the sign ALWAYS to prevent an endless recursion on
+	 * pow() in libc.
+	 */
+	/* quick fix up */
+	DPSIGN(x) ^= 1;
+
 	if (xc == IEEE754_CLASS_SNAN) {
+		ieee754dp y = ieee754dp_indef();
 		SETCX(IEEE754_INVALID_OPERATION);
-		return ieee754dp_nanxcpt(ieee754dp_indef(), "neg");
+		DPSIGN(y) = DPSIGN(x);
+		return ieee754dp_nanxcpt(y, "neg");
 	}
 
 	if (ieee754dp_isnan(x))	/* but not infinity */
 		return ieee754dp_nanxcpt(x, "neg", x);
-
-	/* quick fix up */
-	DPSIGN(x) ^= 1;
 	return x;
 }
 
diff -u linux-mips/arch/mips/math-emu/sp_simple.c linux/arch/mips/math-emu/sp_simple.c
--- linux-mips/arch/mips/math-emu/sp_simple.c	2005-01-31 11:05:18.000000000 +0900
+++ linux/arch/mips/math-emu/sp_simple.c	2005-04-20 17:02:13.678391113 +0900
@@ -48,16 +48,22 @@
 	CLEARCX;
 	FLUSHXSP;
 
+	/*
+	 * Invert the sign ALWAYS to prevent an endless recursion on
+	 * pow() in libc.
+	 */
+	/* quick fix up */
+	SPSIGN(x) ^= 1;
+
 	if (xc == IEEE754_CLASS_SNAN) {
+		ieee754sp y = ieee754sp_indef();
 		SETCX(IEEE754_INVALID_OPERATION);
-		return ieee754sp_nanxcpt(ieee754sp_indef(), "neg");
+		SPSIGN(y) = SPSIGN(x);
+		return ieee754sp_nanxcpt(y, "neg");
 	}
 
 	if (ieee754sp_isnan(x))	/* but not infinity */
 		return ieee754sp_nanxcpt(x, "neg", x);
-
-	/* quick fix up */
-	SPSIGN(x) ^= 1;
 	return x;
 }
 

---
Atsushi Nemoto