Attached is a further revision of the patch, now dealing with check-c++.
Roughly 50% speedup here at '-j32' (18m vs 12m). For my setup
(--enable-languages=c,c++,fortran) I have now improved all targets called in
'make -j32 -k check'. The latter is now 30% faster (15m vs 20m). Note that
there are +- 1m fluctuations in these numbers, easily.
I currently have no plans to work on other check targets before this patch is
committed.
OK for trunk ?
Joost
contrib/ChangeLog
2014-09-09 Joost VandeVondele <vond...@gcc.gnu.org>
* generate_tcl_patterns.sh: New file.
gcc/fortran/ChangeLog
2014-09-09 Joost VandeVondele <vond...@gcc.gnu.org>
* Make-lang.in (check_gfortran_parallelize): Improved parallelism.
gcc/Changelog
2014-09-09 Joost VandeVondele <vond...@gcc.gnu.org>
* Makefile.in (check_gcc_parallelize): Improved parallelism.
(check_p_numbers): Increase maximum value.
(dg_target_exps): Mention targets as separate words only.
(null,space,comma,dg_target_exps_p1,dg_target_exps_p2,
dg_target_exps_p3,dg_target_exps_p4): New variables.
gcc/cp/ChangeLog
2014-09-09 Joost VandeVondele <vond...@gcc.gnu.org>
* Make-lang.in (check_g++_parallelize): Improved parallelism.
libstdc++-v3/ChangeLog
2014-09-09 Joost VandeVondele <vond...@gcc.gnu.org>
* testsuite/Makefile.am (check_DEJAGNU_normal_targets): Add
check-DEJAGNUnormal[11-15].
(check-DEJAGNU): Split into 15 jobs for parallel testing.
* testsuite/Makefile.in: Regenerated.
Index: libstdc++-v3/testsuite/Makefile.am
===================================================================
--- libstdc++-v3/testsuite/Makefile.am (revision 215017)
+++ libstdc++-v3/testsuite/Makefile.am (working copy)
@@ -101,7 +101,7 @@ new-abi-baseline:
@test ! -f $*/site.exp || mv $*/site.exp $*/site.bak
@mv $*/site.exp.tmp $*/site.exp
-check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10)
+check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)
$(check_DEJAGNU_normal_targets): check-DEJAGNUnormal%: normal%/site.exp
# Run the testsuite in normal mode.
@@ -111,7 +111,7 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
if [ -z "$*$(filter-out --target_board=%, $(RUNTESTFLAGS))" ] \
&& [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \
$(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_normal_targets); \
- for idx in 0 1 2 3 4 5 6 7 8 9 10; do \
+ for idx in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do \
mv -f normal$$idx/libstdc++.sum normal$$idx/libstdc++.sum.sep; \
mv -f normal$$idx/libstdc++.log normal$$idx/libstdc++.log.sep; \
done; \
@@ -138,25 +138,35 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
fi; \
dirs="`cd $$srcdir; echo [013-9][0-9]_*/*`";; \
normal1) \
- dirs="`cd $$srcdir; echo [ab]* de* [ep]*/*`";; \
+ dirs="`cd $$srcdir; echo e*/*`";; \
normal2) \
- dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
+ dirs="`cd $$srcdir; echo 28_*/a*`";; \
normal3) \
- dirs="`cd $$srcdir; echo 22_*/*`";; \
+ dirs="`cd $$srcdir; echo 23_*/[lu]*`";; \
normal4) \
- dirs="`cd $$srcdir; echo 23_*/[a-km-tw-z]*`";; \
+ dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
normal5) \
- dirs="`cd $$srcdir; echo 23_*/[luv]*`";; \
+ dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
normal6) \
- dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
+ dirs="`cd $$srcdir; echo 23_*/[m-tw-z]*`";; \
normal7) \
- dirs="`cd $$srcdir; echo 26_*/* 28_*/[c-z]*`";; \
+ dirs="`cd $$srcdir; echo 26_*/*`";; \
normal8) \
dirs="`cd $$srcdir; echo 27_*/*`";; \
normal9) \
- dirs="`cd $$srcdir; echo 28_*/[ab]*`";; \
+ dirs="`cd $$srcdir; echo 22_*/*`";; \
normal10) \
dirs="`cd $$srcdir; echo t*/*`";; \
+ normal11) \
+ dirs="`cd $$srcdir; echo 28_*/b*`";; \
+ normal12) \
+ dirs="`cd $$srcdir; echo 28_*/[c-z]*`";; \
+ normal13) \
+ dirs="`cd $$srcdir; echo de* p*/*`";; \
+ normal14) \
+ dirs="`cd $$srcdir; echo [ab]* 23_*/v*`";; \
+ normal15) \
+ dirs="`cd $$srcdir; echo 23_*/[a-k]*`";; \
esac; \
if [ -n "$*" ]; then cd "$*"; fi; \
if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \
Index: libstdc++-v3/testsuite/Makefile.in
===================================================================
--- libstdc++-v3/testsuite/Makefile.in (revision 215017)
+++ libstdc++-v3/testsuite/Makefile.in (working copy)
@@ -301,7 +301,7 @@ lists_of_files = \
extract_symvers = $(glibcxx_builddir)/scripts/extract_symvers
baseline_subdir := $(shell $(CXX) $(baseline_subdir_switch))
-check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10)
+check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)
# Runs the testsuite, but in compile only mode.
# Can be used to test sources with non-GNU FE's at various warning
@@ -562,7 +562,7 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
if [ -z "$*$(filter-out --target_board=%, $(RUNTESTFLAGS))" ] \
&& [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \
$(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_normal_targets); \
- for idx in 0 1 2 3 4 5 6 7 8 9 10; do \
+ for idx in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do \
mv -f normal$$idx/libstdc++.sum normal$$idx/libstdc++.sum.sep; \
mv -f normal$$idx/libstdc++.log normal$$idx/libstdc++.log.sep; \
done; \
@@ -589,25 +589,35 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
fi; \
dirs="`cd $$srcdir; echo [013-9][0-9]_*/*`";; \
normal1) \
- dirs="`cd $$srcdir; echo [ab]* de* [ep]*/*`";; \
+ dirs="`cd $$srcdir; echo e*/*`";; \
normal2) \
- dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
+ dirs="`cd $$srcdir; echo 28_*/a*`";; \
normal3) \
- dirs="`cd $$srcdir; echo 22_*/*`";; \
+ dirs="`cd $$srcdir; echo 23_*/[lu]*`";; \
normal4) \
- dirs="`cd $$srcdir; echo 23_*/[a-km-tw-z]*`";; \
+ dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
normal5) \
- dirs="`cd $$srcdir; echo 23_*/[luv]*`";; \
+ dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
normal6) \
- dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
+ dirs="`cd $$srcdir; echo 23_*/[m-tw-z]*`";; \
normal7) \
- dirs="`cd $$srcdir; echo 26_*/* 28_*/[c-z]*`";; \
+ dirs="`cd $$srcdir; echo 26_*/*`";; \
normal8) \
dirs="`cd $$srcdir; echo 27_*/*`";; \
normal9) \
- dirs="`cd $$srcdir; echo 28_*/[ab]*`";; \
+ dirs="`cd $$srcdir; echo 22_*/*`";; \
normal10) \
dirs="`cd $$srcdir; echo t*/*`";; \
+ normal11) \
+ dirs="`cd $$srcdir; echo 28_*/b*`";; \
+ normal12) \
+ dirs="`cd $$srcdir; echo 28_*/[c-z]*`";; \
+ normal13) \
+ dirs="`cd $$srcdir; echo de* p*/*`";; \
+ normal14) \
+ dirs="`cd $$srcdir; echo [ab]* 23_*/v*`";; \
+ normal15) \
+ dirs="`cd $$srcdir; echo 23_*/[a-k]*`";; \
esac; \
if [ -n "$*" ]; then cd "$*"; fi; \
if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \
Index: contrib/generate_tcl_patterns.sh
===================================================================
--- contrib/generate_tcl_patterns.sh (revision 0)
+++ contrib/generate_tcl_patterns.sh (revision 0)
@@ -0,0 +1,114 @@
+#! /bin/sh
+
+#
+# based on a list of filenames as input, starting with [0-9A-Za-z],
+# generate regexps that match subsets trying to not exceed a
+# 'maxcount' parameter. Most useful to generate the
+# check_LANG_parallelize assignments needed to split
+# testsuite directories, defining prefix appropriately.
+#
+# Example usage:
+# cd gcc/gcc/testsuite/gfortran.dg
+# ls -1 | ../../../contrib/generate_tcl_patterns.sh 300 "dg.exp=gfortran.dg/"
+#
+# the first parameter is the maximum number of files.
+# the second parameter the prefix used for printing.
+#
+
+# Copyright (C) 2014 Free Software Foundation
+# Contributed by Joost VandeVondele <joost.vandevond...@mat.ethz.ch>
+#
+# This file is part of GCC.
+#
+# GCC is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GCC is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING. If not, write to
+# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+
+gawk -v maxcount=$1 -v prefix=$2 '
+BEGIN{
+ # list of allowed starting chars for a file name in a dir to split
+ achars="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+ ranget="11111111112222222222222222222222222233333333333333333333333333"
+}
+{
+ if (index(achars,substr($1,1,1))==0){
+ print "file : " $1 " does not start with an allowed character."
+ _assert_exit = 1
+ exit 1
+ }
+ nfiles++ ; files[nfiles]=$1
+}
+END{
+ if (_assert_exit) exit 1
+ for(i=1; i<=length(achars); i++) count[substr(achars,i,1)]=0
+ for(i=1; i<=nfiles; i++) {
+ if (length(files[i]>0)) { count[substr(files[i],1,1)]++ }
+ };
+ asort(count,ordered)
+ countsingle=0
+ groups=0
+ label=""
+ for(i=length(achars);i>=1;i--) {
+ countsingle=countsingle+ordered[i]
+ for(j=1;j<=length(achars);j++) {
+ if(count[substr(achars,j,1)]==ordered[i]) found=substr(achars,j,1)
+ }
+ count[found]=-1
+ label=label found
+ if(i==1) { val=maxcount+1 } else { val=ordered[i-1] }
+ if(countsingle+val>maxcount) {
+ subset[label]=countsingle
+ print "Adding label: ", label, "matching files:" countsingle
+ groups++
+ countsingle=0
+ label=""
+ }
+ }
+ print "patterns:"
+ asort(subset,ordered)
+ for(i=groups;i>=1;i--) {
+ for(j in subset){
+ if(subset[j]==ordered[i]) found=j
+ }
+ subset[found]=-1
+ if (length(found)==1) {
+ printf("%s%s* \\\n",prefix,found)
+ } else {
+ sortandcompress()
+ printf("%s\\[%s\\]* \\\n",prefix,found)
+ }
+ }
+}
+function sortandcompress(i,n,tmp,bestj)
+{
+ n=length(found)
+ for(i=1; i<=n; i++) tmp[i]=substr(found,i,1)
+ asort(tmp)
+ for(i=1;i<=n;i++){
+ ipos=index(achars,tmp[i])
+ for(j=i;j<=n;j++){
+ jpos=index(achars,tmp[j])
+ if (jpos-ipos==j-i && substr(ranget,ipos,1)==substr(ranget,jpos,1)) bestj=j
+ }
+ if (bestj-i>3) {
+ tmp[i+1]="-"
+ for(j=i+2;j<bestj;j++) tmp[j]=" "
+ }
+ i=bestj
+ }
+ found=""
+ for(i=1; i<=n; i++) found=found tmp[i]
+ gsub(/ /,"",found)
+}
+'
Index: gcc/fortran/Make-lang.in
===================================================================
--- gcc/fortran/Make-lang.in (revision 215017)
+++ gcc/fortran/Make-lang.in (working copy)
@@ -168,12 +168,22 @@ check-fortran-subtargets : check-gfortra
lang_checks += check-gfortran
lang_checks_parallelized += check-gfortran
# For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_gfortran_parallelize = dg.exp=gfortran.dg/\[adAD\]* \
- dg.exp=gfortran.dg/\[bcBC\]* \
- dg.exp=gfortran.dg/\[nopNOP\]* \
- dg.exp=gfortran.dg/\[isuvISUV\]* \
- dg.exp=gfortran.dg/\[efhkqrxzEFHKQRXZ\]* \
- dg.exp=gfortran.dg/\[0-9gjlmtwyGJLMTWY\]*
+check_gfortran_parallelize = execute.exp \
+ dg.exp=gfortran.dg/p* \
+ dg.exp=gfortran.dg/c* \
+ dg.exp=gfortran.dg/a* \
+ dg.exp=gfortran.dg/i* \
+ dg.exp=gfortran.dg/\[glow\]* \
+ dg.exp=gfortran.dg/\[mu\]* \
+ dg.exp=gfortran.dg/d* \
+ dg.exp=gfortran.dg/s* \
+ dg.exp=gfortran.dg/b* \
+ dg.exp=gfortran.dg/t* \
+ dg.exp=gfortran.dg/f* \
+ dg.exp=gfortran.dg/e* \
+ dg.exp=gfortran.dg/r* \
+ dg.exp=gfortran.dg/n* \
+ dg.exp=gfortran.dg/\[0-9A-Zhjkqvxyz\]*
# GFORTRAN documentation.
GFORTRAN_TEXI = \
Index: gcc/Makefile.in
===================================================================
--- gcc/Makefile.in (revision 215017)
+++ gcc/Makefile.in (working copy)
@@ -513,11 +513,17 @@ xm_include_list=@xm_include_list@
xm_defines=@xm_defines@
lang_checks=
lang_checks_parallelized=
-dg_target_exps:=aarch64.exp,alpha.exp,arm.exp,avr.exp,bfin.exp,cris.exp
-dg_target_exps:=$(dg_target_exps),epiphany.exp,frv.exp,i386.exp,ia64.exp
-dg_target_exps:=$(dg_target_exps),m68k.exp,microblaze.exp,mips.exp,powerpc.exp
-dg_target_exps:=$(dg_target_exps),rx.exp,s390.exp,sh.exp,sparc.exp,spu.exp
-dg_target_exps:=$(dg_target_exps),tic6x.exp,xstormy16.exp
+dg_target_exps:=aarch64 alpha arm avr bfin cris epiphany frv i386 ia64
+dg_target_exps:=$(dg_target_exps) m68k microblaze mips powerpc rx s390
+dg_target_exps:=$(dg_target_exps) sh sparc spu tic6x xstormy16
+# also parallelize the target exps, using one common choice (i386) of split
+null:=
+space:=$(null) $(null)
+comma:=$(null),$(null)
+dg_target_exps_p1:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/a*))
+dg_target_exps_p2:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[fs\]*))
+dg_target_exps_p3:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[0-9A-Zbcdeg-oqrt-z\]*))
+dg_target_exps_p4:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/p*))
# This lists a couple of test files that take most time during check-gcc.
# When doing parallelized check-gcc, these can run in parallel with the
# remaining tests. Each word in this variable stands for work for one
@@ -526,21 +532,42 @@ dg_target_exps:=$(dg_target_exps),tic6x.
# should be run in the same runtest invocation (usually if they aren't
# very long running, but still should be split of from the check-parallel-$lang
# remaining tests runtest invocation), they should be concatenated with commas.
+# At most $(check_p_numbers) goals should be created.
# Note that [a-zA-Z] wildcards need to have []s prefixed with \ (needed
# by tcl) and as the *.exp arguments are mached both as is and with
# */ prefixed to it in runtest_file_p, it is usually desirable to include
# a subdirectory name.
-check_gcc_parallelize=execute.exp=execute/2* \
- execute.exp=execute/\[013-9a-fA-F\]* \
- execute.exp=execute/\[pP\]*,dg.exp \
- execute.exp=execute/\[g-oq-zG-OQ-Z\]*,compile.exp=compile/2* \
- compile.exp=compile/\[9pP\]*,builtins.exp \
- compile.exp=compile/\[013-8a-oq-zA-OQ-Z\]* \
- dg-torture.exp,ieee.exp \
- vect.exp,unsorted.exp \
- guality.exp \
- struct-layout-1.exp,stackalign.exp \
- $(dg_target_exps)
+check_gcc_parallelize=$(dg_target_exps_p1) \
+ $(dg_target_exps_p2) \
+ $(dg_target_exps_p3) \
+ $(dg_target_exps_p4) \
+ execute.exp=execute/2* \
+ execute.exp=execute/p* \
+ execute.exp=execute/9* \
+ execute.exp=execute/\[013-8A-Za-oq-z\]* \
+ compile.exp=compile/p* \
+ compile.exp=compile/2* \
+ compile.exp=compile/\[Babcdfilmsvx\]* \
+ compile.exp=compile/9* \
+ compile.exp=compile/\[013-8AC-Zeghjknoqrtuwyz\]* \
+ dg-torture.exp=torture/p* \
+ dg-torture.exp=torture/\[0-9A-Za-oq-z\]* \
+ vect.exp \
+ atomic.exp \
+ builtins.exp \
+ ubsan.exp \
+ struct-layout-1.exp \
+ stackalign.exp \
+ dg-exp.exp \
+ ieee.exp \
+ asan.exp,tsan.exp \
+ lto.exp \
+ cilk-plus.exp \
+ noncompile.exp,dfp.exp,graphite.exp \
+ guality.exp,unsorted.exp \
+ debug.exp,tree-ssa.exp \
+ pch.exp,compat.exp,cpp.exp \
+ tls.exp,abi-x86_64.exp
lang_opt_files=@lang_opt_files@ $(srcdir)/c-family/c.opt $(srcdir)/common.opt
lang_specs_files=@lang_specs_files@
lang_tree_files=@lang_tree_files@
@@ -3639,7 +3666,8 @@ check_p_vars=$(check_$(check_p_tool)_par
check_p_subno=$(word 2,$(subst _, ,$*))
check_p_comma=,
check_p_subwork=$(subst $(check_p_comma), ,$(if $(check_p_subno),$(word $(check_p_subno),$(check_p_vars))))
-check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
+check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 \
+ 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
check_p_subdir=$(subst _,,$*)
check_p_subdirs=$(wordlist 1,$(words $(check_$*_parallelize)),$(check_p_numbers))
Index: gcc/cp/Make-lang.in
===================================================================
--- gcc/cp/Make-lang.in (revision 215017)
+++ gcc/cp/Make-lang.in (working copy)
@@ -156,10 +156,30 @@ check-c++-subtargets : check-g++-subtarg
lang_checks += check-g++
lang_checks_parallelized += check-g++
# For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_g++_parallelize = old-deja.exp \
- dg.exp=g++.dg/[0-9A-Za-bd-su-z]* \
- dg.exp=g++.dg/[ct]* \
- dg.exp=c-c++-common/*,dg-torture.exp
+check_g++_parallelize = dg.exp=g++.dg/\[ac\]* \
+ dg.exp=g++.dg/\[eint\]* \
+ dg.exp=g++.dg/\[Wflmru\]* \
+ dg.exp=g++.dg/\[dsv\]* \
+ dg.exp=g++.dg/p* \
+ dg.exp=g++.dg/\[0-9A-VXYZbghjkoqwxyz\]* \
+ dg.exp=c-c++-common/* \
+ old-deja.exp=g++.old-deja/g++.p* \
+ old-deja.exp=g++.old-deja/g++.\[lm\]* \
+ old-deja.exp=g++.old-deja/g++.o* \
+ old-deja.exp=g++.old-deja/g++.b* \
+ old-deja.exp=g++.old-deja/g++.\[0-9A-Zac-iknq-z\]* \
+ old-deja.exp=g++.old-deja/g++.j* \
+ dg-torture.exp=torture/p* \
+ dg-torture.exp=torture/\[0-9A-Za-oq-z\]* \
+ asan.exp \
+ ubsan.exp \
+ cilk-plus.exp \
+ struct-layout-1.exp \
+ debug.exp \
+ dwarf2.exp \
+ lto.exp \
+ gomp.exp \
+ tsan.exp
#
# Install hooks:
