ptsname_r, ptsname: Avoid test failures on AIX 7.2

[Bruno Haible]

This patch fixes test failures on AIX 7.1 and 7.2:

FAIL: test-ptsname
==================

FAIL test-ptsname (exit status: 142)

FAIL: test-ptsname_r
====================

FAIL test-ptsname_r (exit status: 142)


2021-01-05  Bruno Haible  <br...@clisp.org>

        ptsname_r, ptsname: Avoid test failures on AIX 7.2.
        * tests/test-ptsname_r.c (main): On AIX, set a 20-seconds timer.
        * tests/test-ptsname.c (main): Likewise. On AIX, open a pty through
        /dev/ptc and don't test the BSD ptys.

diff --git a/tests/test-ptsname.c b/tests/test-ptsname.c
index 70039c9..5dde071 100644
--- a/tests/test-ptsname.c
+++ b/tests/test-ptsname.c
@@ -59,7 +59,11 @@ main (void)
 #if HAVE_DECL_ALARM
   /* Declare failure if test takes too long, by using default abort
      caused by SIGALRM.  */
+# if defined _AIX
+  int alarm_value = 20;
+# else
   int alarm_value = 5;
+# endif
   signal (SIGALRM, SIG_DFL);
   alarm (alarm_value);
 #endif
@@ -120,6 +124,36 @@ main (void)
     close (fd);
   }
 
+#elif defined _AIX
+  /* AIX has BSD-style /dev/ptyp[0-9a-f] files, and the function ptsname()
+     produces the corresponding /dev/ttyp[0-9a-f] file for each.  But opening
+     such a pty causes the process to hang in a state where it does not even
+     react to the SIGALRM signal for N * 15 seconds, where N is the number of
+     opened ptys, either in the close (fd) call or - when this close (fd) call
+     is commented out - at the process exit.
+     So, better don't use these BSD-style ptys.  The modern way to open a pty
+     is to go through /dev/ptc.  */
+  {
+    int fd;
+    char *result;
+
+    /* Open a pty master.  */
+    fd = open ("/dev/ptc", O_RDWR | O_NOCTTY);
+    if (fd < 0)
+      {
+        fprintf (stderr, "Skipping test: cannot open pseudo-terminal\n");
+        return 77;
+      }
+
+    result = ptsname (fd);
+    ASSERT (result != NULL);
+    ASSERT (memcmp (result, "/dev/pts/", 9) == 0);
+
+    /* This close (fd) call takes 15 seconds.  It would be interruptible by the
+       SIGALRM timer, but then this test would report failure.  */
+    close (fd);
+  }
+
 #else
 
   /* Try various master names of Mac OS X: /dev/pty[p-w][0-9a-f]  */
diff --git a/tests/test-ptsname_r.c b/tests/test-ptsname_r.c
index 7622776..21551d2 100644
--- a/tests/test-ptsname_r.c
+++ b/tests/test-ptsname_r.c
@@ -109,7 +109,11 @@ main (void)
 #if HAVE_DECL_ALARM
   /* Declare failure if test takes too long, by using default abort
      caused by SIGALRM.  */
+# if defined _AIX
+  int alarm_value = 20;
+# else
   int alarm_value = 5;
+# endif
   signal (SIGALRM, SIG_DFL);
   alarm (alarm_value);
 #endif
@@ -173,8 +177,14 @@ main (void)
   }
 
 #elif defined _AIX
-  /* AIX has BSD-style /dev/ptyp[0-9a-f] files, but the modern way to open
-     a pty is to go through /dev/ptc.  */
+  /* AIX has BSD-style /dev/ptyp[0-9a-f] files, and the function ptsname()
+     produces the corresponding /dev/ttyp[0-9a-f] file for each.  But opening
+     such a pty causes the process to hang in a state where it does not even
+     react to the SIGALRM signal for N * 15 seconds, where N is the number of
+     opened ptys, either in the close (fd) call or - when this close (fd) call
+     is commented out - at the process exit.
+     So, better don't use these BSD-style ptys.  The modern way to open a pty
+     is to go through /dev/ptc.  */
   {
     int fd;
     char buffer[256];
@@ -194,7 +204,8 @@ main (void)
 
     test_errors (fd, buffer);
 
-    /* This call hangs on AIX.  */
+    /* This close (fd) call takes 15 seconds.  It would be interruptible by the
+       SIGALRM timer, but then this test would report failure.  */
     close (fd);
   }