Pádraig Brady wrote:
> Just to summarize what's happening here...
>
> There are 3 CPU counts possible:
>
> total >= online >= available
>
> "online" corresponds to the CPUs enabled system wide,
> whereas "available" is what's available to a particular
> process which may be less due to affinity config.
>
> "online" is not currently exposed through nproc
> but for reference is got on linux using:
> $ strace -e open getconf _NPROCESSORS_ONLN
> open("/proc/stat"
> for_each_online_cpu() //available to scheduler
>
> "available" on linux is determined using:
> nproc
> glibc::sched_getaffinity()
> return corresponding_syscall();
>
> "total" on linux is determined using:
> nproc --all
> glibc::sysconf(_NPROCESSORS_CONF)
> __get_nprocs_conf ()
> {
> int result = 1;
> /* XXX Here will come a test for the new system call. */
> if (open("/sys/devices/system/cpu"))
> result = parse();
> else if (open("/proc/cpuinfo"))
> result = parse();
> return result;
> }
>
> The last one above is giving the issue as
> it relies on /sys or /proc being available.
Ah! Thanks for the detailed explanation.
> So what about a possible work around?
> How about doing this in nproc(1):
>
> if (num_processors(NPROC_ALL) == 1)
> return num_processors(NPROC_CURRENT_OVERRIDABLE)
It should be NPROC_CURRENT, not NPROC_CURRENT_OVERRIDABLE, I think,
because NPROC_CURRENT_OVERRIDABLE can be overridden by the user quite
arbitrarily.
I like this workaround, but would prefer to have it in the nproc module
in gnulib. coreutils/src/nproc.c is purely the command-line program.
Here's a proposed patch.
2010-01-10 Bruno Haible <br...@clisp.org>
nproc: Work better on Linux when /proc and /sys are not mounted.
* lib/nproc.c (num_processors_via_affinity_mask): New function,
extracted from num_processors.
(num_processors): Call it. Use it as lower bound when, on glibc/Linux
systems, sysconf (_SC_NPROCESSORS_CONF) returns 1.
Suggested by Pádraig Brady <p...@draigbrady.com>.
Reported by Dmitry V. Levin <l...@altlinux.org>.
*** lib/nproc.c.orig Sun Jan 10 11:24:18 2010
--- lib/nproc.c Sun Jan 10 11:21:40 2010
***************
*** 59,229 ****
#define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0]))
! unsigned long int
! num_processors (enum nproc_query query)
{
! if (query == NPROC_CURRENT_OVERRIDABLE)
! {
! /* Test the environment variable OMP_NUM_THREADS, recognized also by all
! programs that are based on OpenMP. The OpenMP spec says that the
! value assigned to the environment variable "may have leading and
! trailing white space". */
! const char *envvalue = getenv ("OMP_NUM_THREADS");
!
! if (envvalue != NULL)
! {
! while (*envvalue != '\0' && c_isspace (*envvalue))
! envvalue++;
! /* Convert it from decimal to 'unsigned long'. */
! if (c_isdigit (*envvalue))
! {
! char *endptr = NULL;
! unsigned long int value = strtoul (envvalue, &endptr, 10);
!
! if (endptr != NULL)
! {
! while (*endptr != '\0' && c_isspace (*endptr))
! endptr++;
! if (*endptr == '\0')
! return (value > 0 ? value : 1);
! }
! }
! }
!
! query = NPROC_CURRENT;
! }
! /* Here query is one of NPROC_ALL, NPROC_CURRENT. */
!
! if (query == NPROC_CURRENT)
! {
! /* glibc >= 2.3.3 with NPTL and NetBSD 5 have pthread_getaffinity_np,
! but with different APIs. Also it requires linking with -lpthread.
! Therefore this code is not enabled.
! glibc >= 2.3.4 has sched_getaffinity whereas NetBSD 5 has
! sched_getaffinity_np. */
#if HAVE_PTHREAD_AFFINITY_NP && defined __GLIBC__ && 0
! {
! cpu_set_t set;
! if (pthread_getaffinity_np (pthread_self (), sizeof (set), &set) == 0)
! {
! unsigned long count;
# ifdef CPU_COUNT
! /* glibc >= 2.6 has the CPU_COUNT macro. */
! count = CPU_COUNT (&set);
# else
! size_t i;
! count = 0;
! for (i = 0; i < CPU_SETSIZE; i++)
! if (CPU_ISSET (i, &set))
! count++;
# endif
! if (count > 0)
! return count;
! }
}
#elif HAVE_PTHREAD_AFFINITY_NP && defined __NetBSD__ && 0
{
! cpuset_t *set;
! set = cpuset_create ();
! if (set != NULL)
{
! unsigned long count = 0;
! if (pthread_getaffinity_np (pthread_self (), cpuset_size (set),
set)
! == 0)
{
! cpuid_t i;
!
! for (i = 0;; i++)
! {
! int ret = cpuset_isset (i, set);
! if (ret < 0)
! break;
! if (ret > 0)
! count++;
! }
}
- cpuset_destroy (set);
- if (count > 0)
- return count;
}
}
#elif HAVE_SCHED_GETAFFINITY_LIKE_GLIBC /* glibc >= 2.3.4 */
! {
! cpu_set_t set;
! if (sched_getaffinity (0, sizeof (set), &set) == 0)
! {
! unsigned long count;
# ifdef CPU_COUNT
! /* glibc >= 2.6 has the CPU_COUNT macro. */
! count = CPU_COUNT (&set);
# else
! size_t i;
! count = 0;
! for (i = 0; i < CPU_SETSIZE; i++)
! if (CPU_ISSET (i, &set))
! count++;
# endif
! if (count > 0)
! return count;
! }
}
#elif HAVE_SCHED_GETAFFINITY_NP /* NetBSD >= 5 */
{
! cpuset_t *set;
! set = cpuset_create ();
! if (set != NULL)
{
! unsigned long count = 0;
! if (sched_getaffinity_np (getpid (), cpuset_size (set), set) == 0)
{
! cpuid_t i;
!
! for (i = 0;; i++)
! {
! int ret = cpuset_isset (i, set);
! if (ret < 0)
! break;
! if (ret > 0)
! count++;
! }
}
- cpuset_destroy (set);
- if (count > 0)
- return count;
}
}
#endif
#if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
! { /* This works on native Windows platforms. */
! DWORD_PTR process_mask;
! DWORD_PTR system_mask;
! if (GetProcessAffinityMask (GetCurrentProcess (),
! &process_mask, &system_mask))
! {
! DWORD_PTR mask = process_mask;
! unsigned long count = 0;
! for (; mask != 0; mask = mask >> 1)
! if (mask & 1)
! count++;
! if (count > 0)
! return count;
! }
}
#endif
#if defined _SC_NPROCESSORS_ONLN
{ /* This works on glibc, MacOS X 10.5, FreeBSD, AIX, OSF/1, Solaris,
Cygwin, Haiku. */
--- 59,261 ----
#define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0]))
! /* Return the number of processors available to the current process, based
! on a modern system call that returns the "affinity" between the current
! process and each CPU. Return 0 if unknown or if such a system call does
! not exist. */
! static unsigned long
! num_processors_via_affinity_mask (void)
{
! /* glibc >= 2.3.3 with NPTL and NetBSD 5 have pthread_getaffinity_np,
! but with different APIs. Also it requires linking with -lpthread.
! Therefore this code is not enabled.
! glibc >= 2.3.4 has sched_getaffinity whereas NetBSD 5 has
! sched_getaffinity_np. */
#if HAVE_PTHREAD_AFFINITY_NP && defined __GLIBC__ && 0
! {
! cpu_set_t set;
! if (pthread_getaffinity_np (pthread_self (), sizeof (set), &set) == 0)
! {
! unsigned long count;
# ifdef CPU_COUNT
! /* glibc >= 2.6 has the CPU_COUNT macro. */
! count = CPU_COUNT (&set);
# else
! size_t i;
! count = 0;
! for (i = 0; i < CPU_SETSIZE; i++)
! if (CPU_ISSET (i, &set))
! count++;
# endif
! if (count > 0)
! return count;
}
+ }
#elif HAVE_PTHREAD_AFFINITY_NP && defined __NetBSD__ && 0
+ {
+ cpuset_t *set;
+
+ set = cpuset_create ();
+ if (set != NULL)
{
! unsigned long count = 0;
! if (pthread_getaffinity_np (pthread_self (), cpuset_size (set), set)
! == 0)
{
! cpuid_t i;
! for (i = 0;; i++)
{
! int ret = cpuset_isset (i, set);
! if (ret < 0)
! break;
! if (ret > 0)
! count++;
}
}
+ cpuset_destroy (set);
+ if (count > 0)
+ return count;
}
+ }
#elif HAVE_SCHED_GETAFFINITY_LIKE_GLIBC /* glibc >= 2.3.4 */
! {
! cpu_set_t set;
! if (sched_getaffinity (0, sizeof (set), &set) == 0)
! {
! unsigned long count;
# ifdef CPU_COUNT
! /* glibc >= 2.6 has the CPU_COUNT macro. */
! count = CPU_COUNT (&set);
# else
! size_t i;
! count = 0;
! for (i = 0; i < CPU_SETSIZE; i++)
! if (CPU_ISSET (i, &set))
! count++;
# endif
! if (count > 0)
! return count;
}
+ }
#elif HAVE_SCHED_GETAFFINITY_NP /* NetBSD >= 5 */
+ {
+ cpuset_t *set;
+
+ set = cpuset_create ();
+ if (set != NULL)
{
! unsigned long count = 0;
! if (sched_getaffinity_np (getpid (), cpuset_size (set), set) == 0)
{
! cpuid_t i;
! for (i = 0;; i++)
{
! int ret = cpuset_isset (i, set);
! if (ret < 0)
! break;
! if (ret > 0)
! count++;
}
}
+ cpuset_destroy (set);
+ if (count > 0)
+ return count;
}
+ }
#endif
#if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
! { /* This works on native Windows platforms. */
! DWORD_PTR process_mask;
! DWORD_PTR system_mask;
! if (GetProcessAffinityMask (GetCurrentProcess (),
! &process_mask, &system_mask))
! {
! DWORD_PTR mask = process_mask;
! unsigned long count = 0;
! for (; mask != 0; mask = mask >> 1)
! if (mask & 1)
! count++;
! if (count > 0)
! return count;
}
+ }
#endif
+ return 0;
+ }
+
+ unsigned long int
+ num_processors (enum nproc_query query)
+ {
+ if (query == NPROC_CURRENT_OVERRIDABLE)
+ {
+ /* Test the environment variable OMP_NUM_THREADS, recognized also by all
+ programs that are based on OpenMP. The OpenMP spec says that the
+ value assigned to the environment variable "may have leading and
+ trailing white space". */
+ const char *envvalue = getenv ("OMP_NUM_THREADS");
+
+ if (envvalue != NULL)
+ {
+ while (*envvalue != '\0' && c_isspace (*envvalue))
+ envvalue++;
+ /* Convert it from decimal to 'unsigned long'. */
+ if (c_isdigit (*envvalue))
+ {
+ char *endptr = NULL;
+ unsigned long int value = strtoul (envvalue, &endptr, 10);
+
+ if (endptr != NULL)
+ {
+ while (*endptr != '\0' && c_isspace (*endptr))
+ endptr++;
+ if (*endptr == '\0')
+ return (value > 0 ? value : 1);
+ }
+ }
+ }
+
+ query = NPROC_CURRENT;
+ }
+ /* Here query is one of NPROC_ALL, NPROC_CURRENT. */
+
+ /* On systems with a modern affinity mask system call, we have
+ sysconf (_SC_NPROCESSORS_CONF)
+ >= sysconf (_SC_NPROCESSORS_ONLN)
+ >= num_processors_via_affinity_mask ()
+ The first number is the number of CPUs configured in the system.
+ The second number is the number of CPUs available to the scheduler.
+ The third number is the number of CPUs available to the current process.
+
+ Note! On Linux systems with glibc, the first and second number come from
+ the /sys and /proc file systems (see
+ glibc/sysdeps/unix/sysv/linux/getsysstats.c).
+ In some situations these file systems are not mounted, and the sysconf
+ call returns 1, which does not reflect the reality. */
+
+ if (query == NPROC_CURRENT)
+ {
+ /* Try the modern affinity mask system call. */
+ {
+ unsigned long nprocs = num_processors_via_affinity_mask ();
+
+ if (nprocs > 0)
+ return nprocs;
+ }
+
#if defined _SC_NPROCESSORS_ONLN
{ /* This works on glibc, MacOS X 10.5, FreeBSD, AIX, OSF/1, Solaris,
Cygwin, Haiku. */
***************
*** 239,244 ****
--- 271,292 ----
{ /* This works on glibc, MacOS X 10.5, FreeBSD, AIX, OSF/1, Solaris,
Cygwin, Haiku. */
long int nprocs = sysconf (_SC_NPROCESSORS_CONF);
+
+ # if __GLIBC__ >= 2 && defined __linux__
+ /* On Linux systems with glibc, this information comes from the /sys
and
+ /proc file systems (see
glibc/sysdeps/unix/sysv/linux/getsysstats.c).
+ In some situations these file systems are not mounted, and the
+ sysconf call returns 1. But we wish to guarantee that
+ num_processors (NPROC_ALL) >= num_processors (NPROC_CURRENT). */
+ if (nprocs == 1)
+ {
+ unsigned long nprocs_current = num_processors_via_affinity_mask
();
+
+ if (nprocs_current > 0)
+ nprocs = nprocs_current;
+ }
+ # endif
+
if (nprocs > 0)
return nprocs;
}
