On 06/26/2017 01:24 PM, Tom de Vries wrote:
Hi,
I've written a patch series to facilitate debugging libgomp openacc
testcase failures on the nvptx accelerator.
When running an openacc test-case on an nvptx accelerator, the following
happens:
- the plugin obtains the ptx assembly for the acceleration kernels
- it calls the cuda jit to compile and link the ptx into a module
- it loads the module
- it starts an acceleration kernel
The patch series adds these environment variables:
- GOMP_OPENACC_NVPTX_SAVE_TEMPS: a means to save the resulting module
such that it can be investigated using nvdisasm and cuobjdump.
- GOMP_OPENACC_NVPTX_DISASM: a means to see the resulting module in
the debug output, by writing it into a file and calling nvdisasm on
it
- GOMP_OPENACC_NVPTX_JIT: a means to set parameters of the
compilation/linking process, currently supporting:
* -O[0-4], mapping onto CU_JIT_OPTIMIZATION_LEVEL
* -ori, mapping onto CU_JIT_NEW_SM3X_OPT
The patch series consists of these patches:
2. Handle GOMP_OPENACC_NVPTX_{DISASM,SAVE_TEMPS} in libgomp nvptx plugin
This patch adds handling of:
- GOMP_OPENACC_NVPTX_SAVE_TEMPS=[01], and
- GOMP_OPENACC_NVPTX_DISASM=[01]
The filename used for dumping the module is plugin-nvptx.<pid>.cubin.
Thanks,
- Tom
Handle GOMP_OPENACC_NVPTX_{DISASM,SAVE_TEMPS} in libgomp nvptx plugin
2017-06-26 Tom de Vries <t...@codesourcery.com>
* plugin/plugin-nvptx.c (do_prog, debug_linkout): New function.
(link_ptx): Use debug_linkout.
---
libgomp/plugin/plugin-nvptx.c | 103 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 103 insertions(+)
diff --git a/libgomp/plugin/plugin-nvptx.c b/libgomp/plugin/plugin-nvptx.c
index 71630b5..df1bfdd 100644
--- a/libgomp/plugin/plugin-nvptx.c
+++ b/libgomp/plugin/plugin-nvptx.c
@@ -47,6 +47,9 @@
#include <unistd.h>
#include <assert.h>
#include <errno.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/wait.h>
#if PLUGIN_NVPTX_DYNAMIC
# include <dlfcn.h>
@@ -876,6 +879,104 @@ notify_var (const char *var_name, const char *env_var)
GOMP_PLUGIN_debug (0, "%s: '%s'\n", var_name, env_var);
}
+static void
+do_prog (const char *prog, const char *arg)
+{
+ pid_t pid = fork ();
+
+ if (pid == -1)
+ {
+ GOMP_PLUGIN_error ("Fork failed");
+ return;
+ }
+ else if (pid > 0)
+ {
+ int status;
+ waitpid (pid, &status, 0);
+ if (!WIFEXITED (status))
+ GOMP_PLUGIN_error ("Running %s %s failed", prog, arg);
+ }
+ else
+ {
+ execlp (prog, prog /* argv[0] */, arg, NULL);
+ abort ();
+ }
+}
+
+static void
+debug_linkout (void *linkout, size_t linkoutsize)
+{
+ static int gomp_openacc_nvptx_disasm = -1;
+ if (gomp_openacc_nvptx_disasm == -1)
+ {
+ const char *var_name = "GOMP_OPENACC_NVPTX_DISASM";
+ const char *env_var = getenv (var_name);
+ notify_var (var_name, env_var);
+ gomp_openacc_nvptx_disasm
+ = ((env_var != NULL && env_var[0] == '1' && env_var[1] == '\0')
+ ? 1 : 0);
+ }
+
+ static int gomp_openacc_nvptx_save_temps = -1;
+ if (gomp_openacc_nvptx_save_temps == -1)
+ {
+ const char *var_name = "GOMP_OPENACC_NVPTX_SAVE_TEMPS";
+ const char *env_var = getenv (var_name);
+ notify_var (var_name, env_var);
+ gomp_openacc_nvptx_save_temps
+ = ((env_var != NULL && env_var[0] == '1' && env_var[1] == '\0')
+ ? 1 : 0);
+ }
+
+ if (gomp_openacc_nvptx_disasm == 0
+ && gomp_openacc_nvptx_save_temps == 0)
+ return;
+
+ const char *prefix = "plugin-nvptx.";
+ const char *postfix = ".cubin";
+ const int len = (strlen (prefix)
+ + 20 /* %lld. */
+ + strlen (postfix)
+ + 1 /* '\0'. */);
+ char file_name[len];
+ int res = snprintf (file_name, len, "%s%lld%s", prefix,
+ (long long)getpid (), postfix);
+ assert (res < len); /* Assert there's no truncation. */
+
+ GOMP_PLUGIN_debug (0, "Generating %s with size %zu\n",
+ file_name, linkoutsize);
+ FILE *cubin_file = fopen (file_name, "wb");
+ if (cubin_file == NULL)
+ {
+ GOMP_PLUGIN_debug (0, "Opening %s failed\n", file_name);
+ return;
+ }
+
+ fwrite (linkout, linkoutsize, 1, cubin_file);
+ unsigned int write_succeeded = ferror (cubin_file) == 0;
+ if (!write_succeeded)
+ GOMP_PLUGIN_debug (0, "Writing %s failed\n", file_name);
+
+ res = fclose (cubin_file);
+ if (res != 0)
+ GOMP_PLUGIN_debug (0, "Closing %s failed\n", file_name);
+
+ if (!write_succeeded)
+ return;
+
+ if (gomp_openacc_nvptx_disasm == 1)
+ {
+ GOMP_PLUGIN_debug (0, "Disassembling %s\n", file_name);
+ do_prog ("nvdisasm", file_name);
+ }
+
+ if (gomp_openacc_nvptx_save_temps == 0)
+ {
+ GOMP_PLUGIN_debug (0, "Removing %s\n", file_name);
+ remove (file_name);
+ }
+}
+
static bool
link_ptx (CUmodule *module, const struct targ_ptx_obj *ptx_objs,
unsigned num_objs)
@@ -939,6 +1040,8 @@ link_ptx (CUmodule *module, const struct targ_ptx_obj *ptx_objs,
return false;
}
+ debug_linkout (linkout, linkoutsize);
+
CUDA_CALL (cuModuleLoadData, module, linkout);
CUDA_CALL (cuLinkDestroy, linkstate);
return true;
