fix various bugs in new integer parser framework

1. my interpretation of subject sequence definition was wrong. adjust
parser to conform to the standard.

2. some code for handling tail overflow case was missing (forgot to
finish writing it).

3. typo (= instead of ==) caused ERANGE to wrongly behave like EINVAL

src/internal/intparse.c

@@ -35,6 +35,7 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 	v->cnt += n;
 	for (; n; n--, s++) switch (v->state) {
 	case 0:
+		v->err = EINVAL;
 		v->state++;
 		if (*s=='+' || *s=='-') {
 			v->neg = *s=='-';
@@ -49,6 +50,7 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 	case 2:
 		v->state++;
 		if ((!b || b==16) && (*s|32) == 'x') {
+			v->err = 0;
 			v->base = b = 16;
 			continue;
 		}
@@ -57,10 +59,11 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 	case 3:
 	firstdigit:
 		if (digits[*s] >= b) {
-			v->err = EINVAL;
-			return 0;
+			n++;
+			goto finished;
 		}
 	seconddigit:
+		v->err = 0;
 		v->state++;
 	case 4:
 		if (b==10) {
@@ -92,11 +95,11 @@ int __intparse(struct intparse *v, const void *buf, size_t n)
 		if (n && digits[*s]<b) {
 			v->err = ERANGE;
 			v->val = UINTMAX_MAX;
-
 			n--; s++;
+			for (; n && digits[*s]<b; n--, s++);
 		}
-		for (; n && digits[*s]<b; n--, s++);
 		if (!n) return 1;
+		goto finished;
 	}
 	return 1;
 finished:

src/stdlib/strtoimax.c

@@ -25,7 +25,7 @@ intmax_t strtoimax(const char *s1, char **p, int base)
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return ip.neg ? INTMAX_MIN : INTMAX_MAX;
 	}
 

src/stdlib/strtoumax.c

@@ -26,7 +26,7 @@ uintmax_t strtoumax(const char *s1, char **p, int base)
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return UINTMAX_MAX;
 	}
 

src/stdlib/wcstoimax.c

@@ -6,6 +6,7 @@
 
 intmax_t wcstoimax(const wchar_t *s, wchar_t **p, int base)
 {
+	const wchar_t *s1 = s;
 	struct intparse ip = {0};
 
 	if (p) *p = (wchar_t *)s;
@@ -21,11 +22,11 @@ intmax_t wcstoimax(const wchar_t *s, wchar_t **p, int base)
 	for (; __intparse(&ip, (char[]){(*s&-(*s<128U))}, 1); s++);
 
 	if (p && ip.err != EINVAL)
-		*p = (wchar_t *)s;
+		*p = (wchar_t *)s1 + ip.cnt;
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return ip.neg ? INTMAX_MIN : INTMAX_MAX;
 	}
 

src/stdlib/wcstoumax.c

@@ -7,6 +7,7 @@
 
 uintmax_t wcstoumax(const wchar_t *s, wchar_t **p, int base)
 {
+	const wchar_t *s1 = s;
 	struct intparse ip = {0};
 
 	if (p) *p = (wchar_t *)s;
@@ -22,11 +23,11 @@ uintmax_t wcstoumax(const wchar_t *s, wchar_t **p, int base)
 	for (; __intparse(&ip, (char[]){(*s&-(*s<128U))}, 1); s++);
 
 	if (p && ip.err != EINVAL)
-		*p = (wchar_t *)s;
+		*p = (wchar_t *)s1 + ip.cnt;
 
 	if (ip.err) {
 		errno = ip.err;
-		if (ip.err = EINVAL) return 0;
+		if (ip.err == EINVAL) return 0;
 		return UINTMAX_MAX;
 	}