use prlimit syscall for getrlimit/setrlimit

this allows the full range of 64-bit limit arguments even on 32-bit
systems. fallback to the old syscalls on old kernels that don't
support prlimit.

src/misc/getrlimit.c

@@ -1,14 +1,18 @@
 #include <sys/resource.h>
+#include <errno.h>
 #include "syscall.h"
 #include "libc.h"
 
 int getrlimit(int resource, struct rlimit *rlim)
 {
-	long k_rlim[2];
+	unsigned long k_rlim[2];
+	int ret = syscall(SYS_prlimit64, 0, resource, 0, rlim);
+	if (!ret || errno != ENOSYS)
+		return ret;
 	if (syscall(SYS_getrlimit, resource, k_rlim) < 0)
 		return -1;
-	rlim->rlim_cur = k_rlim[0] == -1 ? -1 : (unsigned long)k_rlim[0];
-	rlim->rlim_max = k_rlim[1] == -1 ? -1 : (unsigned long)k_rlim[1];
+	rlim->rlim_cur = k_rlim[0] == -1UL ? RLIM_INFINITY : k_rlim[0];
+	rlim->rlim_max = k_rlim[1] == -1UL ? RLIM_INFINITY : k_rlim[1];
 	return 0;
 }
 

src/misc/setrlimit.c

@@ -3,9 +3,15 @@
 #include "syscall.h"
 #include "libc.h"
 
+#define MIN(a, b) ((a)<(b) ? (a) : (b))
+
 int __setrlimit(int resource, const struct rlimit *rlim)
 {
-	long k_rlim[2] = { rlim->rlim_cur, rlim->rlim_max };
+	unsigned long k_rlim[2];
+	int ret = __syscall(SYS_prlimit64, 0, resource, rlim, 0);
+	if (ret != -ENOSYS) return ret;
+	k_rlim[0] = MIN(rlim->rlim_cur, -1UL);
+	k_rlim[1] = MIN(rlim->rlim_max, -1UL);
 	return __syscall(SYS_setrlimit, resource, k_rlim);
 }