[PATCH] posix: fix system when a child cannot be created [BZ #32450]

POSIX states that "if a child process cannot be created, or if the
termination status for the command language interpreter cannot be
obtained, system() shall return -1 and set errno to indicate the error."

In the glibc implementation it could happen when posix_spawn fails,
which happens when the underlying fork, vfork, or clone call fails. They
could fail with EAGAIN and ENOMEM.

Resolves: BZ #32450
Signed-off-by: Aurelien Jarno <aurelien@aurel32.net>
Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
Gbp-Pq: Topic any
Gbp-Pq: Name git-posix-fix-system-when-a-child-cannot-be-created-BZ-3.patch

diff --git a/stdlib/tst-system.c b/stdlib/tst-system.c
index 47c742f963a1b6b4ac7980e16830d3f460e89eb7..7ed332e2ae5532f5034874f0809f322a38a7efcc 100644 (file)
--- a/stdlib/tst-system.c
+++ b/stdlib/tst-system.c
@@ -20,6 +20,7 @@
 #include <string.h>
 #include <signal.h>
 #include <paths.h>
+#include <sys/resource.h>
 
 #include <support/capture_subprocess.h>
 #include <support/check.h>
@@ -194,6 +195,26 @@ do_test (void)
     xpthread_join (long_sleep_thread);
   }
 
+  {
+    struct rlimit rlimit_orig, rlimit_new;
+
+    if (getrlimit (RLIMIT_NPROC, &rlimit_orig) != 0)
+      FAIL_EXIT1 ("getrlimit (RLIMIT_NPROC) failed: %m");
+
+    /* Force failure for the system call */
+    rlimit_new.rlim_cur = 0;
+    rlimit_new.rlim_max = rlimit_orig.rlim_max;
+
+    if (setrlimit (RLIMIT_NPROC, &rlimit_new) != 0)
+      FAIL_EXIT1 ("setrlimit (RLIMIT_NPROC) failed: %m");
+
+    TEST_COMPARE (system (""), -1);
+
+    /* Restore NPROC limit */
+    if (setrlimit (RLIMIT_NPROC, &rlimit_orig) != 0)
+      FAIL_EXIT1 ("setrlimit (RLIMIT_NPROC) failed: %m");
+  }
+
   TEST_COMPARE (system (""), 0);
 
   return 0;

diff --git a/sysdeps/posix/system.c b/sysdeps/posix/system.c
index 01e8bc45215bb289ad32efa3231ea0290e75896c..7f04d03b426007ab63a248bf7fc25856f2f06321 100644 (file)
--- a/sysdeps/posix/system.c
+++ b/sysdeps/posix/system.c
@@ -175,10 +175,14 @@ do_system (const char *line)
       __libc_cleanup_region_end (0);
 #endif
     }
+  else if (ret == EAGAIN || ret == ENOMEM)
+    /* POSIX states that failure to create a child process should
+       return -1.  */
+    status = -1;
   else
-   /* POSIX states that failure to execute the shell should return
-      as if the shell had terminated using _exit(127).  */
-   status = W_EXITCODE (127, 0);
+    /* POSIX states that failure to execute the shell should return
+       as if the shell had terminated using _exit(127).  */
+    status = W_EXITCODE (127, 0);
 
   /* sigaction can not fail with SIGINT/SIGQUIT used with old
      disposition.  Same applies for sigprocmask.  */