This patch adds "binary mode" handling for popen() on platforms where the native popen() will not. This is a problem especially on platforms where fopen() will ignore the 'b' but popen() will not. Hence, the test for "does fopen() accept a 'b'" won't apply to popen(). GNU libc ought to just ignore the 'b', but it doesn't.
2010-02-21 Bruce Korb <bk...@gnu.org>
* m4/popen.m4: check to see if popen() can handle a "rb" mode string. * lib/popen.c (rpl_popen): strip out any 'b' from the mode string. * lib/popen-safer.c (popen_safer): ditto --- lib/popen.c 2010-02-17 16:03:53.000000000 -0800 +++ lib/popen-new.c 2010-02-22 09:45:11.000000000 -0800 @@ -47,6 +47,30 @@ rpl_popen (const char *filename, const c int cloexec1 = fcntl (STDOUT_FILENO, F_GETFD); int saved_errno; +#if ! BINARY_MODE_POPEN + char tmp_mode[8], *tmp_pz = tmp_mode; + + for (;;) + { + int ch = *(mode++); + *tmp_pz = ch; + if (ch == '\0') + break; + if (ch != 'b') { + tmp_pz++; + + /* Silently truncate overlong mode strings. Let the popen() funciton + complain about the issue. It will. */ + if (tmp_pz == tmp_mode + sizeof (tmp_mode) - 1) + { + *tmp_pz = '\0'; + break; + } + } + } + mode = tmp_mode; +#endif + /* If either stdin or stdout was closed (that is, fcntl failed), then we open a dummy close-on-exec fd to occupy that slot. That way, popen's internal use of pipe() will not contain either fd 0 --- lib/popen-safer.c 2010-02-17 16:03:53.000000000 -0800 +++ lib/popen-safer-new.c 2010-02-22 09:47:01.000000000 -0800 @@ -73,6 +73,7 @@ popen_safer (char const *cmd, char const the original fd created by popen is safe. */ FILE *fp; int fd = open_noinherit ("/dev/null", O_RDONLY); + if (0 <= fd && fd <= STDERR_FILENO) { /* Maximum recursion depth is 3. */ @@ -84,6 +85,30 @@ popen_safer (char const *cmd, char const } else { +#if ! BINARY_MODE_POPEN + char tmp_mode[8], *tmp_pz = tmp_mode; + + for (;;) + { + int ch = *(mode++); + *tmp_pz = ch; + if (ch == '\0') + break; + if (ch != 'b') { + tmp_pz++; + + /* Silently truncate overlong mode strings. Let the popen() + funciton complain about the issue. It will. */ + if (tmp_pz == tmp_mode + sizeof (tmp_mode) - 1) + { + *tmp_pz = '\0'; + break; + } + } + } + mode = tmp_mode; +#endif + /* Either all fd's are tied up, or fd is safe and the real popen will reuse it. */ close (fd); --- m4/popen.m4 2010-02-17 16:03:53.000000000 -0800 +++ m4/popen-new.m4 2010-02-22 09:36:56.000000000 -0800 @@ -25,6 +25,12 @@ AC_DEFUN([gl_FUNC_POPEN], AC_LIBOBJ([popen]) gl_PREREQ_POPEN fi + AC_RUN_IFELSE( + [AC_LANG_PROGRAM([], [dnl + FILE * fp = popen ("date", "rb"); + exit (fp == NULL);])], + [AC_DEFINE([BINARY_MODE_POPEN], [1], [define if popen supports "rb" mode])] + ) ]) # Prerequisites of lib/popen.c.