greatly improve freopen behavior

1. don't open /dev/null just as a basis to copy flags; use shared
__fmodeflags function to get the right file flags for the mode.

2. handle the case (probably invalid, but whatever) case where the
original stream's file descriptor was closed; previously, the logic
re-closed it.

3. accept the "e" mode flag for close-on-exec; update dup3 to fallback
to using dup2 so we can simply call __dup3 instead of putting fallback
logic in freopen itself.

src/internal/stdio_impl.h

@@ -87,6 +87,7 @@ size_t __fwritex(const unsigned char *, size_t, FILE *);
 int __putc_unlocked(int, FILE *);
 
 FILE *__fdopen(int, const char *);
+int __fmodeflags(const char *);
 
 #define OFLLOCK() LOCK(libc.ofl_lock)
 #define OFLUNLOCK() UNLOCK(libc.ofl_lock)

src/stdio/__fmodeflags.c

@@ -0,0 +1,16 @@
+#include <fcntl.h>
+#include <string.h>
+
+int __fmodeflags(const char *mode)
+{
+	int flags;
+	if (strchr(mode, '+')) flags = O_RDWR;
+	else if (*mode == 'r') flags = O_RDONLY;
+	else flags = O_WRONLY;
+	if (strchr(mode, 'x')) flags |= O_EXCL;
+	if (strchr(mode, 'e')) flags |= O_CLOEXEC;
+	if (*mode != 'r') flags |= O_CREAT;
+	if (*mode == 'w') flags |= O_TRUNC;
+	if (*mode == 'a') flags |= O_APPEND;
+	return flags;
+}

src/stdio/fopen.c

@@ -13,14 +13,7 @@ FILE *fopen(const char *restrict filename, const char *restrict mode)
 	}
 
 	/* Compute the flags to pass to open() */
-	if (strchr(mode, '+')) flags = O_RDWR;
+	flags = __fmodeflags(mode);
-	else if (*mode == 'r') flags = O_RDONLY;
-	else flags = O_WRONLY;
-	if (strchr(mode, 'x')) flags |= O_EXCL;
-	if (strchr(mode, 'e')) flags |= O_CLOEXEC;
-	if (*mode != 'r') flags |= O_CREAT;
-	if (*mode == 'w') flags |= O_TRUNC;
-	if (*mode == 'a') flags |= O_APPEND;
 
 	fd = syscall_cp(SYS_open, filename, flags|O_LARGEFILE, 0666);
 	if (fd < 0) return 0;

src/stdio/freopen.c

@@ -7,24 +7,27 @@
 /* Locking is not necessary because, in the event of failure, the stream
  * passed to freopen is invalid as soon as freopen is called. */
 
+int __dup3(int, int, int);
+
 FILE *freopen(const char *restrict filename, const char *restrict mode, FILE *restrict f)
 {
-	int fl;
+	int fl = __fmodeflags(mode);
 	FILE *f2;
 
 	fflush(f);
 
 	if (!filename) {
-		f2 = fopen("/dev/null", mode);
+		if (fl&O_CLOEXEC)
-		if (!f2) goto fail;
+			__syscall(SYS_fcntl, f->fd, F_SETFD, FD_CLOEXEC);
-		fl = __syscall(SYS_fcntl, f2->fd, F_GETFL, 0);
+		fl &= ~(O_CREAT|O_EXCL|O_CLOEXEC);
 		if (syscall(SYS_fcntl, f->fd, F_SETFL, fl) < 0)
-			goto fail2;
+			goto fail;
+		return f;
 	} else {
 		f2 = fopen(filename, mode);
 		if (!f2) goto fail;
-		if (syscall(SYS_dup2, f2->fd, f->fd) < 0)
+		if (f2->fd == f->fd) f2->fd = -1; /* avoid closing in fclose */
-			goto fail2;
+		else if (__dup3(f2->fd, f->fd, fl&O_CLOEXEC)<0) goto fail2;
 	}
 
 	f->flags = (f->flags & F_PERM) | f2->flags;

src/unistd/dup3.c

@@ -1,10 +1,21 @@
 #define _GNU_SOURCE
 #include <unistd.h>
 #include <errno.h>
+#include <fcntl.h>
 #include "syscall.h"
+#include "libc.h"
 
-int dup3(int old, int new, int flags) {
+int __dup3(int old, int new, int flags)
+{
 	int r;
-	while ((r=__syscall(SYS_dup3, old, new, flags))==-EBUSY);
+	if (old==new) return __syscall_ret(-EINVAL);
+	if (flags & O_CLOEXEC) {
+		while ((r=__syscall(SYS_dup3, old, new, flags))==-EBUSY);
+		if (r!=-ENOSYS) return __syscall_ret(r);
+	}
+	while ((r=__syscall(SYS_dup2, old, new))==-EBUSY);
+	if (flags & O_CLOEXEC) __syscall(SYS_fcntl, new, F_SETFD, FD_CLOEXEC);
 	return __syscall_ret(r);
 }
+
+weak_alias(__dup3, dup3);