yaxunl created this revision.
yaxunl added a reviewer: tra.
Herald added subscribers: jansvoboda11, dang, kerbowa, nhaehnle, jvesely.
yaxunl requested review of this revision.
Spack is a package management tool extensively used by HPC community.
As ROCm packages are built by Spack by HPC community, we need to teach
clang driver to detect ROCm installation built by Spack.
https://reviews.llvm.org/D97340
Files:
clang/include/clang/Driver/Options.td
clang/lib/Driver/ToolChains/AMDGPU.cpp
clang/lib/Driver/ToolChains/ROCm.h
clang/test/Driver/Inputs/rocm-spack/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5/bin/.hipVersion
clang/test/Driver/Inputs/rocm-spack/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5/include/hip/hip_runtime.h
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/asanrtl.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/hip.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/ockl.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_correctly_rounded_sqrt_off.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_correctly_rounded_sqrt_on.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_daz_opt_off.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_daz_opt_on.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_finite_only_off.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_finite_only_on.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_isa_version_1010.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_isa_version_1011.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_isa_version_1012.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_isa_version_803.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_isa_version_900.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_isa_version_908.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_unsafe_math_off.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_unsafe_math_on.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_wavefrontsize64_off.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/oclc_wavefrontsize64_on.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/ocml.bc
clang/test/Driver/Inputs/rocm-spack/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/opencl.bc
clang/test/Driver/rocm-detect.hip
Index: clang/test/Driver/rocm-detect.hip
===================================================================
--- clang/test/Driver/rocm-detect.hip
+++ clang/test/Driver/rocm-detect.hip
@@ -16,14 +16,71 @@
// RUN: --rocm-path=%S/Inputs/rocm %s 2>&1 \
// RUN: | FileCheck -check-prefixes=COMMON,GFX902-DEFAULTLIBS %s
-
// RUN: %clang -### -v -target x86_64-linux-gnu --cuda-gpu-arch=gfx902 -nogpulib \
// RUN: --rocm-path=%S/Inputs/rocm %s 2>&1 \
// RUN: | FileCheck -check-prefixes=COMMON,NODEFAULTLIBS %s
+// Test ROCm installation built by SPACK by invoke clang at %T/rocm-spack/llvm-amdgpu-*
+// directory through a soft link.
+
+// RUN: rm -rf %T/rocm-spack
+// RUN: cp -r %S/Inputs/rocm-spack %T
+// RUN: ln -fs %clang %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang
+// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \
+// RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 %s 2>&1 \
+// RUN: | FileCheck -check-prefixes=SPACK %s
+
+// Test SPACK installation with multiple hip and rocm-device-libs packages of the same
+// ROCm release. Clang cannot determine which one to use and emit diagnostics. --hip-path
+// and --rocm-device-lib-path can be used to specify them.
+
+// RUN: cp -r %T/rocm-spack/hip-* %T/rocm-spack/hip-4.0.0-abcd
+// RUN: cp -r %T/rocm-spack/rocm-device-libs-* %T/rocm-spack/rocm-device-libs-4.0.0-efgh
+// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \
+// RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 %s 2>&1 \
+// RUN: | FileCheck -check-prefixes=SPACK-MULT %s
+// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \
+// RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 \
+// RUN: --hip-path=%T/rocm-spack/hip-4.0.0-abcd \
+// RUN: --rocm-device-lib-path=%T/rocm-spack/rocm-device-libs-4.0.0-efgh/amdgcn/bitcode \
+// RUN: %s 2>&1 | FileCheck -check-prefixes=SPACK-SET %s
+
+// Test invalid SPACK ROCm installation missing hip and rocm-device-libs packages.
+
+// RUN: rm -rf %T/rocm-spack/hip-*
+// RUN: rm -rf %T/rocm-spack/rocm-device-libs-*
+// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \
+// RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 %s 2>&1 \
+// RUN: | FileCheck -check-prefixes=SPACK-MISS %s
// GFX902-DEFAULTLIBS: error: cannot find ROCm device library for gfx902. Provide its path via --rocm-path or --rocm-device-lib-path, or pass -nogpulib to build without ROCm device library
// NODEFAULTLIBS-NOT: error: cannot find
// COMMON: "-triple" "amdgcn-amd-amdhsa"
+
+// SPACK: InstalledDir: [[DIR:.*]]/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin
+// SPACK: Found HIP installation: [[DIR]]/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5, version 4.0.20214-a2917cd
+// SPACK: "-triple" "amdgcn-amd-amdhsa"
+// SPACK-SAME: "-mlink-builtin-bitcode" "[[DIR]]/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/hip.bc"
+// SPACK-SAME: "-internal-isystem" "[[DIR]]/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5/include"
+
+// SPACK-MULT: InstalledDir: [[DIR:.*]]/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin
+// SPACK-MULT-DAG: Expecting one SPACK package hip-4.0.0 at [[DIR]] but found more
+// SPACK-MULT-DAG: Expecting one SPACK package rocm-device-libs-4.0.0 at [[DIR]] but found more
+// SPACK-MULT-NOT: Found HIP installation: [[DIR]]/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5, version 4.0.20214-a2917cd
+// SPACK-MULT-NOT: "-mlink-builtin-bitcode" "[[DIR]]/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/hip.bc"
+// SPACK-MULT-NOT: "-internal-isystem" "[[DIR]]/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5/include"
+
+// SPACK-SET: InstalledDir: [[DIR:.*]]/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin
+// SPACK-SET: Found HIP installation: [[DIR]]/hip-4.0.0-abcd, version 4.0.20214-a2917cd
+// SPACK-SET: "-triple" "amdgcn-amd-amdhsa"
+// SPACK-SET-SAME: "-mlink-builtin-bitcode" "[[DIR]]/rocm-device-libs-4.0.0-efgh/amdgcn/bitcode/hip.bc"
+// SPACK-SET-SAME: "-internal-isystem" "[[DIR]]/hip-4.0.0-abcd/include"
+
+// SPACK-MISS: InstalledDir: [[DIR:.*]]/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin
+// SPACK-MISS-DAG: Expecting SPACK package hip-4.0.0 at [[DIR]] but not found
+// SPACK-MISS-DAG: Expecting SPACK package rocm-device-libs-4.0.0 at [[DIR]] but not found
+// SPACK-MISS-NOT: Found HIP installation: [[DIR]]/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5, version 4.0.20214-a2917cd
+// SPACK-MISS-NOT: "-mlink-builtin-bitcode" "[[DIR]]/rocm-device-libs-4.0.0-6wnyzz4hgl3hr7uswasnagt7j2adctbs/amdgcn/bitcode/hip.bc"
+// SPACK-MISS-NOT: "-internal-isystem" "[[DIR]]/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5/include"
Index: clang/test/Driver/Inputs/rocm-spack/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5/bin/.hipVersion
===================================================================
--- /dev/null
+++ clang/test/Driver/Inputs/rocm-spack/hip-4.0.0-5f63slrursbrvfe2txrrjkynbsywsob5/bin/.hipVersion
@@ -0,0 +1,5 @@
+# NOTE: The trailing whitespace is added on purpose to verify that these
+# whitespaces are trimmed before paring.
+HIP_VERSION_MAJOR=4
+HIP_VERSION_MINOR=0
+HIP_VERSION_PATCH=20214-a2917cd
Index: clang/lib/Driver/ToolChains/ROCm.h
===================================================================
--- clang/lib/Driver/ToolChains/ROCm.h
+++ clang/lib/Driver/ToolChains/ROCm.h
@@ -47,6 +47,15 @@
: Path(Path), StrictChecking(StrictChecking) {}
};
+ struct CandidatesInfo {
+ // Candidates for ROCm root directory.
+ SmallVector<Candidate, 4> Candidates;
+ // Release string for ROCm packages built with SPACK if not empty. The
+ // installation directories of ROCm packages built with SPACK follow the
+ // convention <package_name>-<rocm_release_string>-<commit_hash>.
+ llvm::SmallString<0> SPACKReleaseStr;
+ };
+
const Driver &D;
bool HasHIPRuntime = false;
bool HasDeviceLibrary = false;
@@ -67,6 +76,8 @@
StringRef RocmPathArg;
// ROCm device library paths specified by --rocm-device-lib-path.
std::vector<std::string> RocmDeviceLibPathArg;
+ // HIP runtime path specified by --hip-path.
+ StringRef HIPPathArg;
// HIP version specified by --hip-version.
StringRef HIPVersionArg;
// Wheter -nogpulib is specified.
@@ -107,7 +118,7 @@
void scanLibDevicePath(llvm::StringRef Path);
bool parseHIPVersionFile(llvm::StringRef V);
- SmallVector<Candidate, 4> getInstallationPathCandidates();
+ CandidatesInfo getInstallationPathCandidates();
public:
RocmInstallationDetector(const Driver &D, const llvm::Triple &HostTriple,
Index: clang/lib/Driver/ToolChains/AMDGPU.cpp
===================================================================
--- clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -22,6 +22,42 @@
using namespace clang;
using namespace llvm::opt;
+// Look for sub-directory starts with Prefix under Path. If there is one and
+// only one matching sub-directory found, append the sub-directory to Path. If
+// there is no matching sub-directory or there are more than one matching
+// sub-directories, diagnose them. Returns true if there is only one matching
+// sub-directory.
+static void handleSPACKPackage(const Driver &D, SmallString<0> &Path,
+ StringRef Prefix) {
+ std::error_code EC;
+ llvm::SmallVector<llvm::SmallString<0>> SubDirs;
+ for (llvm::vfs::directory_iterator File = D.getVFS().dir_begin(Path, EC),
+ FileEnd;
+ File != FileEnd && !EC; File.increment(EC)) {
+ llvm::StringRef FileName = llvm::sys::path::filename(File->path());
+ if (FileName.startswith(Prefix)) {
+ SubDirs.push_back(FileName);
+ if (SubDirs.size() > 1)
+ break;
+ }
+ }
+ if (SubDirs.size() == 1) {
+ llvm::sys::path::append(Path, SubDirs[0]);
+ }
+ if (SubDirs.size() == 0) {
+ unsigned DiagID = D.getDiags().getCustomDiagID(
+ DiagnosticsEngine::Error,
+ "Expecting SPACK package %0 at %1 but not found");
+ D.Diag(DiagID) << Prefix << Path;
+ }
+ if (SubDirs.size() > 1) {
+ unsigned DiagID = D.getDiags().getCustomDiagID(
+ DiagnosticsEngine::Error,
+ "Expecting one SPACK package %0 at %1 but found more");
+ D.Diag(DiagID) << Prefix << Path;
+ }
+}
+
void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) {
assert(!Path.empty());
@@ -117,12 +153,12 @@
}
// For candidate specified by --rocm-path we do not do strict check.
-SmallVector<RocmInstallationDetector::Candidate, 4>
+RocmInstallationDetector::CandidatesInfo
RocmInstallationDetector::getInstallationPathCandidates() {
- SmallVector<Candidate, 4> Candidates;
+ CandidatesInfo CanInfo;
if (!RocmPathArg.empty()) {
- Candidates.emplace_back(RocmPathArg.str());
- return Candidates;
+ CanInfo.Candidates.emplace_back(RocmPathArg.str());
+ return CanInfo;
}
// Try to find relative to the compiler binary.
@@ -146,17 +182,34 @@
if (ParentName == "llvm")
ParentDir = llvm::sys::path::parent_path(ParentDir);
- Candidates.emplace_back(ParentDir.str(), /*StrictChecking=*/true);
+ // Detect ROCm packages built with SPACK.
+ // clang is installed at
+ // <rocm_root>/llvm-amdgpu-<rocm_release_string>-<hash>/bin directory.
+ // We only consider the parent directory of llvm-amdgpu package as ROCm
+ // installation candidate for SPACK.
+ if (ParentName.startswith("llvm-amdgpu-")) {
+ auto SPACKReleaseStr =
+ ParentName.drop_front(strlen("llvm-amdgpu-")).split('-').first;
+ if (!SPACKReleaseStr.empty()) {
+ ParentDir = llvm::sys::path::parent_path(ParentDir);
+ CanInfo.SPACKReleaseStr = SPACKReleaseStr;
+ CanInfo.Candidates.emplace_back(ParentDir.str(), /*StrictChecking=*/true);
+ return CanInfo;
+ }
+ }
+
+ CanInfo.Candidates.emplace_back(ParentDir.str(), /*StrictChecking=*/true);
// Device library may be installed in clang resource directory.
- Candidates.emplace_back(D.ResourceDir, /*StrictChecking=*/true);
+ CanInfo.Candidates.emplace_back(D.ResourceDir, /*StrictChecking=*/true);
- Candidates.emplace_back(D.SysRoot + "/opt/rocm", /*StrictChecking=*/true);
+ CanInfo.Candidates.emplace_back(D.SysRoot + "/opt/rocm",
+ /*StrictChecking=*/true);
if (ParentName.startswith("aomp")) {
// Some versions of the aomp package install to /opt/rocm/aomp/bin
- Candidates.emplace_back(ParentDir.str());
+ CanInfo.Candidates.emplace_back(ParentDir.str());
}
- return Candidates;
+ return CanInfo;
}
RocmInstallationDetector::RocmInstallationDetector(
@@ -166,6 +219,7 @@
RocmPathArg = Args.getLastArgValue(clang::driver::options::OPT_rocm_path_EQ);
RocmDeviceLibPathArg =
Args.getAllArgValues(clang::driver::options::OPT_rocm_device_lib_path_EQ);
+ HIPPathArg = Args.getLastArgValue(clang::driver::options::OPT_hip_path_EQ);
if (auto *A = Args.getLastArg(clang::driver::options::OPT_hip_version_EQ)) {
HIPVersionArg = A->getValue();
unsigned Major = 0;
@@ -228,8 +282,8 @@
// exist for each frontend project, and differ depending on which build
// system produced the packages. Standalone OpenCL builds also have a
// different directory structure from the ROCm OpenCL package.
- auto Candidates = getInstallationPathCandidates();
- for (const auto &Candidate : Candidates) {
+ auto CanInfo = getInstallationPathCandidates();
+ for (const auto &Candidate : CanInfo.Candidates) {
auto CandidatePath = Candidate.Path;
// Check device library exists at the given path.
@@ -267,6 +321,12 @@
// Make a path by appending sub-directories to InstallPath.
auto MakePath = [&](const llvm::ArrayRef<const char *> &SubDirs) {
auto Path = CandidatePath;
+ // Device library built by SPACK is installed to
+ // <rocm_root>/rocm-device-libs-<rocm_release_string>-<hash> directory.
+ if (!CanInfo.SPACKReleaseStr.empty())
+ handleSPACKPackage(
+ D, Path,
+ llvm::Twine("rocm-device-libs-" + CanInfo.SPACKReleaseStr).str());
for (auto SubDir : SubDirs)
llvm::sys::path::append(Path, SubDir);
return Path;
@@ -282,13 +342,22 @@
}
void RocmInstallationDetector::detectHIPRuntime() {
- auto Candidates = getInstallationPathCandidates();
+ CandidatesInfo CanInfo;
+ if (!HIPPathArg.empty())
+ CanInfo.Candidates.emplace_back(HIPPathArg.str(), /*StrictChecking=*/true);
+ else
+ CanInfo = getInstallationPathCandidates();
auto &FS = D.getVFS();
- for (const auto &Candidate : Candidates) {
+ for (const auto &Candidate : CanInfo.Candidates) {
InstallPath = Candidate.Path;
if (InstallPath.empty() || !FS.exists(InstallPath))
continue;
+ // HIP runtime built by SPACK is installed to
+ // <rocm_root>/hip-<rocm_release_string>-<hash> directory.
+ if (!CanInfo.SPACKReleaseStr.empty())
+ handleSPACKPackage(D, InstallPath,
+ llvm::Twine("hip-" + CanInfo.SPACKReleaseStr).str());
BinPath = InstallPath;
llvm::sys::path::append(BinPath, "bin");
Index: clang/include/clang/Driver/Options.td
===================================================================
--- clang/include/clang/Driver/Options.td
+++ clang/include/clang/Driver/Options.td
@@ -901,6 +901,8 @@
[], TargetOpts<"NVPTXUseShortPointers">>;
def rocm_path_EQ : Joined<["--"], "rocm-path=">, Group<i_Group>,
HelpText<"ROCm installation path, used for finding and automatically linking required bitcode libraries.">;
+def hip_path_EQ : Joined<["--"], "hip-path=">, Group<i_Group>,
+ HelpText<"HIP runtime installation path, used for finding HIP version and adding HIP include path.">;
def rocm_device_lib_path_EQ : Joined<["--"], "rocm-device-lib-path=">, Group<Link_Group>,
HelpText<"ROCm device library path. Alternative to rocm-path.">;
def : Joined<["--"], "hip-device-lib-path=">, Alias<rocm_device_lib_path_EQ>;
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
