[slurm-dev] FAILURE on slurm_submit_batch_job due to execve

[Stephen Barrett]

FAILURE happens because the job returns a non-zero value. In this case, an
8. According to the job output it's coming from execve, which is an ENOEXEC
error:

slurmstepd: error: execve(): /var/spool/slurmd/job00436/slurm_script: Exec
format error


A couple of things: the script is task.sh, not slurm_script; and there is
no /var/spool/slurmd directory on the machine; the code works if I pass in
a literal script rather than a file (e.g., "#!/bin/bash\nhostname" in
successful job 459).

Here is the code, and the output for failing job 458:

int main(int argc, char* argv[])
{
  string wkdir("/home/stephenb/testing/slurm");
  char* cwkdir = new char[wkdir.size() + 1];
  strcpy(cwkdir, wkdir.c_str());

  string name("bbones");
  char* cname = new char[name.size() + 1];
  strcpy(cname, name.c_str());

  string script(wkdir + "/task.sh");
  char* cscript = new char[script.size() + 1];
  strcpy(cscript, script.c_str());

  // Copy over current environment.
  vector<string> varv;
  size_t varc(-1);

  while (environ[++varc])
    varv.push_back(environ[varc]);

  char **cvarv = new char*[varc];
  for (size_t i = 0; i < varc; i++)
  {
    cvarv[i] = new char[varv[i].size() + 1];
    strcpy(cvarv[i], varv[i].c_str());
  }


  job_desc_msg_t desc;
  slurm_init_job_desc_msg(&desc);

  desc.env_size = varc;
  desc.environment = cvarv;
  desc.name = cname;
  desc.script = cscript;
  desc.user_id = getuid();
  desc.work_dir = cwkdir;


  cout << "--- job description ---" << endl;
  cout << "   argc=" << desc.argc << endl;
  cout << "   varc=" << desc.env_size << endl;
  cout << "   name=" << desc.name << endl;
  cout << "   script=" << desc.script << endl;
  cout << "   user_id=" << desc.user_id << endl;
  cout << "   work_dir=" << desc.work_dir << endl;


  submit_response_msg_t* srsp;
  int rc = slurm_submit_batch_job(&desc, &srsp);

  if (rc == 0)
  {
    cout << "--- job response ---" << endl;
    cout << "   job_id=" << srsp->job_id << endl;
    cout << "   step_id=" << srsp->step_id << endl;
    cout << "   error_code=" << srsp->error_code << endl;


    job_info_msg_t* info;
    rc = slurm_load_job(&info, srsp->job_id, 0);

    if (rc == 0)
    {
      job_info_t* detail = info->job_array;

      cout << "--- job detail ---" << endl << "   " << flush;
      FILE* flog = fopen("/dev/stdout", "w");
      slurm_print_job_info(flog, detail, 0);
      slurm_free_job_info_msg(info);
    }

    slurm_free_submit_response_response_msg(srsp);
  }

  delete cname;
  delete cscript;
  delete cwkdir;
  delete[] cvarv;

  return EXIT_SUCCESS;
}


--- job description ---
   argc=0
   varc=45
   name=bbones
   script=/home/stephenb/testing/slurm/task.sh
   user_id=517
   workdir=/home/stephenb/testing/slurm

--- job response --
   job_id=458
   step_id=4294967294
   error_code=0

--- job detail ---
   JobId=458 JobName=bbones
   UserId=stephenb(517) GroupId=root(0) MCS_label=N/A
   Priority=1753 Nice=0 Account=totalcae QOS=normal
   JobState=PENDING Reason=None Dependency=(null)
   Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0
   DerivedExitCode=0:0
   RunTime=00:00:00 TimeLimit=UNLIMITED TimeMin=N/A
   SubmitTime=2016-09-02T09:33:01 EligibleTime=2016-09-02T09:33:01
   StartTime=Unknown EndTime=Unknown Deadline=N/A
   PreemptTime=None SuspendTime=None SecsPreSuspend=0
   Partition=gen2-128-24 AllocNode:Sid=I96:20916
   ReqNodeList=(null) ExcNodeList=(null)
   NodeList=(null)
   NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:*
   TRES=cpu=1,node=1
   Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=*
   MinCPUsNode=1 MinMemoryNode=0 MinTmpDiskNode=0
   Features=(null) Gres=(null) Reservation=(null)
   OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null)
   Command=(null)
   WorkDir=/home/stephenb/testing/slurm
   StdErr=/home/stephenb/testing/slurm/slurm-458.out
   StdIn=StdIn=/dev/null
   StdOut=/home/stephenb/testing/slurm/slurm-458.out
   BatchScript=
/home/stephenb/testing/slurm/task.sh
   Power=


       JobID      State ExitCode
------------ ---------- --------
458              FAILED      8:0
458.batch        FAILED      8:0
459           COMPLETED      0:0
459.batch     COMPLETED      0:0


I note that the comment for slurm_job_info_t.batch_script reads "contents
of batch script," but here we see the file name, not the contents. When it
actually *is* the script contents, as with job 459, the job completes.

Any ideas as to what's going on here?

-- 
*Steve Barrett*, Sr. Solver Development Engineer
p: 517.319.0512
www.redcedartech.com