atexit: fail rather than deadlocking after last handler is called

previously, global dtors, which are executed after all atexit handlers
have been called rather than being implemented as an atexit handler
themselves, would deadlock if they called atexit.

it was intentional to disallow adding more atexit handlers past the
last point where they would be executed, since a successful return
from atexit imposes a contract that the handler will be executed, but
this was only considered in the context of calls to atexit from other
threads, not calls from the dtors.

to fix this, release the lock after the exit handlers loop completes,
but but set a flag first so that we can make all future calls to
atexit return a failure code.

src/exit/atexit.c

@@ -19,6 +19,7 @@ static struct fl
 	void *a[COUNT];
 } builtin, *head;
 
+static int finished_atexit;
 static int slot;
 static volatile int lock[1];
 volatile int *const __atexit_lockptr = lock;
@@ -34,6 +35,10 @@ void __funcs_on_exit()
 		func(arg);
 		LOCK(lock);
 	}
+	/* Unlock to prevent deadlock if a global dtor
+	 * attempts to call atexit. */
+	finished_atexit = 1;
+	UNLOCK(lock);
 }
 
 void __cxa_finalize(void *dso)
@@ -44,6 +49,13 @@ int __cxa_atexit(void (*func)(void *), void *arg, void *dso)
 {
 	LOCK(lock);
 
+	/* Prevent dtors from registering further atexit
+	 * handlers that would never be run. */
+	if (finished_atexit) {
+		UNLOCK(lock);
+		return -1;
+	}
+
 	/* Defer initialization of head so it can be in BSS */
 	if (!head) head = &builtin;