> On Dec 17, 2015, at 11:37 AM, Justin Bogner <m...@justinbogner.com> wrote: > > Chris Bieneman <be...@apple.com <mailto:be...@apple.com>> writes: >> beanz created this revision. >> beanz added reviewers: bogner, silvas, chandlerc. >> beanz added a subscriber: cfe-commits. >> >> This patch adds support for the clang multi-stage bootstrapping to >> support PGO profdata generation, and can build a 2 or 3 stage >> compiler. >> >> With this patch applied you can configure your build directory with >> the following invocation of CMake: >> >> cmake -G <generator> -C <path_to_clang>/cmake/caches/PGO-stage1.cmake >> <source dir> >> >> After configuration the following additional targets will be generated: >> >> stage2: >> Builds a stage1 x86 compiler, runtime, and required tools >> (llvm-config, llvm-profdata) then uses that compiler to build an >> instrumented stage2 compiler. >> >> stage2-generate-profdata: >> Depends on "stage2" and will use the stage2 compiler to generate >> profdata based on the training files in <clang>/utils/perf-training >> >> stage3: >> Depends on "stage2-generate-profdata" and will use the stage1 compiler >> with the stage2 profdata to build a PGO-optimized compiler. > > Let's bikeshed a bit about terminology here. > > This isn't really what "stage3" means in the typical sense - that term > is generally for a compiler built with a stage 2. What we're getting out > of this process is a PGO optimized compiler, but the result is actually > a stage 2 compiler. If we ever want to build another PGO'd compiler > using this one and do a binary compare, *that* would be stage 3. > > So if we want to use the "stage" terminology we have a stage1, a > training-stage2 and a optimized-stage2, or something like that. Those > are a little long though, so maybe it would make sense to just call > these stage1, training, and stage2. WDYT?
How about stage2-instrumented and just stage2? Prefacing the instrumented with “stage2” lets us know what stage we’re on. That way it will continue to work if someone did either a stage1 or stage3 instrumented build. > > > On a related note, the way these cache files are happening there are > sort of two kinds of them. The stage1 files are more-or-less user facing > and use the later stage cache files in their builds. Maybe we should > drop the stage1 from the user facing files and name them so that they > imply what the total result of the build will be (ie, it's just PGO). I’ll make that change. I’m going to send out updated patches shortly caveat that I am sending them “untested”. The CMake configuration for stage1 works, and I expect it to fully work, but the many-stage builds take a long time. > >> stage3-check-llvm: >> Depends on stage3 and runs check-llvm using the stage3 compiler. >> >> stage3-check-clang: >> Depends on stage3 and runs check-clang using the stage3 compiler. >> >> stage3-check-all: >> Depends on stage3 and runs check-all using the stage3 compiler. >> >> stage3-test-suite: >> Depends on stage3 and runs the test-suite using the stage3 compiler >> (requires in-tree test-suite). >> >> http://reviews.llvm.org/D15584 >> >> Files: >> CMakeLists.txt >> cmake/caches/PGO-stage1.cmake >> cmake/caches/PGO-stage2.cmake >> cmake/caches/PGO-stage3.cmake >> >> Index: cmake/caches/PGO-stage3.cmake >> =================================================================== >> --- /dev/null >> +++ cmake/caches/PGO-stage3.cmake >> @@ -0,0 +1,2 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") > > Does this actually use the generated profdata file? I can't find a place > where it does that. This is actually setup in this diff in clang/CMakelists.txt. Look for PGO_OPT. > >> Index: cmake/caches/PGO-stage2.cmake >> =================================================================== >> --- /dev/null >> +++ cmake/caches/PGO-stage2.cmake >> @@ -0,0 +1,9 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") >> + >> +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite >> CACHE STRING "") >> + >> +set(CLANG_BOOTSTRAP_CMAKE_ARGS >> + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake >> + CACHE STRING "") >> Index: cmake/caches/PGO-stage1.cmake >> =================================================================== >> --- /dev/null >> +++ cmake/caches/PGO-stage1.cmake >> @@ -0,0 +1,17 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") >> + >> +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") >> +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") >> +set(CLANG_BOOTSTRAP_TARGETS >> + stage3 >> + generate-profdata >> + stage3-check-all >> + stage3-check-llvm >> + stage3-check-clang >> + stage3-test-suite CACHE STRING "") >> + >> +set(CLANG_BOOTSTRAP_CMAKE_ARGS >> + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake >> + CACHE STRING "") >> Index: CMakeLists.txt >> =================================================================== >> --- CMakeLists.txt >> +++ CMakeLists.txt >> @@ -677,6 +677,25 @@ >> CLANG_REPOSITORY_STRING >> CMAKE_MAKE_PROGRAM) >> >> + set(COMPILER_OPTIONS >> + -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ >> + -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang >> + -DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) >> + >> + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) >> + set(PGO_DEP llvm-profdata) >> + set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) >> + endif() >> + >> + if(LLVM_BUILD_INSTRUMENTED) >> + set(PGO_DEP generate-profdata) >> + set(PGO_OPT >> -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata) >> + set(COMPILER_OPTIONS >> + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} >> + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} >> + -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}) >> + endif() >> + >> if(TARGET compiler-rt) >> set(RUNTIME_DEP compiler-rt) >> endif() >> @@ -703,7 +722,7 @@ >> endforeach() >> >> ExternalProject_Add(${NEXT_CLANG_STAGE} >> - DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} >> + DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} ${PGO_DEP} >> PREFIX ${NEXT_CLANG_STAGE} >> SOURCE_DIR ${CMAKE_SOURCE_DIR} >> STAMP_DIR ${STAMP_DIR} >> @@ -715,11 +734,9 @@ >> -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} >> ${CLANG_BOOTSTRAP_CMAKE_ARGS} >> ${PASSTHROUGH_VARIABLES} >> - -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ >> - -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang >> - -DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang >> - -DCLANG_STAGE=${NEXT_CLANG_STAGE} >> - ${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} >> + -DCLANG_STAGE=${NEXT_CLANG_STAGE} >> + ${COMPILER_OPTIONS} >> + ${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} ${PGO_OPT} >> INSTALL_COMMAND "" >> STEP_TARGETS configure build >> ${cmake_3_4_USES_TERMINAL_OPTIONS} >> >> >> Index: cmake/caches/PGO-stage3.cmake >> =================================================================== >> --- /dev/null >> +++ cmake/caches/PGO-stage3.cmake >> @@ -0,0 +1,2 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") >> Index: cmake/caches/PGO-stage2.cmake >> =================================================================== >> --- /dev/null >> +++ cmake/caches/PGO-stage2.cmake >> @@ -0,0 +1,9 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") >> + >> +set(CLANG_BOOTSTRAP_TARGETS check-all check-llvm check-clang test-suite >> CACHE STRING "") >> + >> +set(CLANG_BOOTSTRAP_CMAKE_ARGS >> + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage3.cmake >> + CACHE STRING "") >> Index: cmake/caches/PGO-stage1.cmake >> =================================================================== >> --- /dev/null >> +++ cmake/caches/PGO-stage1.cmake >> @@ -0,0 +1,17 @@ >> +set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "") >> +set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "") >> +set(LLVM_BUILD_EXTERNAL_COMPILER_RT ON CACHE BOOL "") >> + >> +set(LLVM_TARGETS_TO_BUILD X86 CACHE STRING "") >> +set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "") >> +set(CLANG_BOOTSTRAP_TARGETS >> + stage3 >> + generate-profdata >> + stage3-check-all >> + stage3-check-llvm >> + stage3-check-clang >> + stage3-test-suite CACHE STRING "") >> + >> +set(CLANG_BOOTSTRAP_CMAKE_ARGS >> + -C ${CMAKE_CURRENT_LIST_DIR}/PGO-stage2.cmake >> + CACHE STRING "") >> Index: CMakeLists.txt >> =================================================================== >> --- CMakeLists.txt >> +++ CMakeLists.txt >> @@ -677,6 +677,25 @@ >> CLANG_REPOSITORY_STRING >> CMAKE_MAKE_PROGRAM) >> >> + set(COMPILER_OPTIONS >> + -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ >> + -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang >> + -DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang) >> + >> + if(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED) >> + set(PGO_DEP llvm-profdata) >> + set(PGO_OPT -DLLVM_PROFDATA=${LLVM_RUNTIME_OUTPUT_INTDIR}/llvm-profdata) >> + endif() >> + >> + if(LLVM_BUILD_INSTRUMENTED) >> + set(PGO_DEP generate-profdata) >> + set(PGO_OPT >> -DLLVM_PROFDATA_FILE=${CMAKE_CURRENT_BINARY_DIR}/utils/perf-training/clang.profdata) >> + set(COMPILER_OPTIONS >> + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} >> + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} >> + -DCMAKE_ASM_COMPILER=${CMAKE_ASM_COMPILER}) >> + endif() >> + >> if(TARGET compiler-rt) >> set(RUNTIME_DEP compiler-rt) >> endif() >> @@ -703,7 +722,7 @@ >> endforeach() >> >> ExternalProject_Add(${NEXT_CLANG_STAGE} >> - DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} >> + DEPENDS clang ${LTO_DEP} ${RUNTIME_DEP} ${PGO_DEP} >> PREFIX ${NEXT_CLANG_STAGE} >> SOURCE_DIR ${CMAKE_SOURCE_DIR} >> STAMP_DIR ${STAMP_DIR} >> @@ -715,11 +734,9 @@ >> -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} >> ${CLANG_BOOTSTRAP_CMAKE_ARGS} >> ${PASSTHROUGH_VARIABLES} >> - -DCMAKE_CXX_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang++ >> - -DCMAKE_C_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang >> - -DCMAKE_ASM_COMPILER=${LLVM_RUNTIME_OUTPUT_INTDIR}/clang >> - -DCLANG_STAGE=${NEXT_CLANG_STAGE} >> - ${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} >> + -DCLANG_STAGE=${NEXT_CLANG_STAGE} >> + ${COMPILER_OPTIONS} >> + ${LTO_LIBRARY} ${LTO_AR} ${LTO_RANLIB} ${verbose} ${PGO_OPT} >> INSTALL_COMMAND "" >> STEP_TARGETS configure build >> ${cmake_3_4_USES_TERMINAL_OPTIONS}
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
