add support for m68k 80-bit long double variant

since x86 and m68k are the only archs with 80-bit long double and each
has mandatory endianness, select the variant via endianness.
differences are minor: apparently just byte order and representation
of infinities. the m68k format is not well-documented anywhere I could
find, so if other differences are found they may require additional
changes later.

src/internal/libm.h

@@ -28,6 +28,17 @@ union ldshape {
 		uint16_t se;
 	} i;
 };
+#elif LDBL_MANT_DIG == 64 && LDBL_MAX_EXP == 16384 && __BYTE_ORDER == __BIG_ENDIAN
+/* This is the m68k variant of 80-bit long double, and this definition only works
+ * on archs where the alignment requirement of uint64_t is <= 4. */
+union ldshape {
+	long double f;
+	struct {
+		uint16_t se;
+		uint16_t pad;
+		uint64_t m;
+	} i;
+};
 #elif LDBL_MANT_DIG == 113 && LDBL_MAX_EXP == 16384 && __BYTE_ORDER == __LITTLE_ENDIAN
 union ldshape {
 	long double f;

src/math/__fpclassifyl.c

@@ -13,10 +13,18 @@ int __fpclassifyl(long double x)
 	int msb = u.i.m>>63;
 	if (!e && !msb)
 		return u.i.m ? FP_SUBNORMAL : FP_ZERO;
+	if (e == 0x7fff) {
+		/* The x86 variant of 80-bit extended precision only admits
+		 * one representation of each infinity, with the mantissa msb
+		 * necessarily set. The version with it clear is invalid/nan.
+		 * The m68k variant, however, allows either, and tooling uses
+		 * the version with it clear. */
+		if (__BYTE_ORDER == __LITTLE_ENDIAN && !msb)
+			return FP_NAN;
+		return u.i.m << 1 ? FP_NAN : FP_INFINITE;
+	}
 	if (!msb)
 		return FP_NAN;
-	if (e == 0x7fff)
-		return u.i.m << 1 ? FP_NAN : FP_INFINITE;
 	return FP_NORMAL;
 }
 #elif LDBL_MANT_DIG == 113 && LDBL_MAX_EXP == 16384