Changeset: 84e9ac87f7fc for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=84e9ac87f7fc
Modified Files:
testing/Mtest.py.in
Branch: Oct2020
Log Message:
Use with clauses to start sub processes when possible.
This makes that they get terminated when the clause exits for whatever
reason (e.g. exception).
diffs (truncated from 571 to 300 lines):
diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in
--- a/testing/Mtest.py.in
+++ b/testing/Mtest.py.in
@@ -1537,10 +1537,10 @@ def ApproveOutput (env, TST) :
patch = os.path.join(os.sep, 'tmp')
patch = os.path.join(patch, "%s.patch-%s" %
(os.path.basename(stableOUTPUT), str(os.getpid())))
f = openutf8(patch + '.0', 'w')
- proc = process.Popen(['diff', '-Bb', '-I^[#=]', '-I^MAPI *=',
'-U0',
- stableOUTPUT, testOUTPUT],
- stdout=f, text=True)
- proc.wait()
+ with process.Popen(['diff', '-Bb', '-I^[#=]', '-I^MAPI *=', '-U0',
+ stableOUTPUT, testOUTPUT],
+ stdout=f, text=True) as proc:
+ proc.wait()
f.close()
if os.path.getsize(patch + ".0"):
# if a file TST.stable.{out,err}-noapprove exists, we
@@ -1582,14 +1582,14 @@ def ApproveOutput (env, TST) :
patchcmd = ['patch']
if not verbose:
patchcmd.append('--quiet')
- proc = process.Popen(patchcmd + [stableOUTPUT, patch + '.1'],
- text=True)
- proc.wait()
+ with process.Popen(patchcmd + [stableOUTPUT, patch + '.1'],
+ text=True) as proc:
+ proc.wait()
f = openutf8(patch, 'w')
- proc = process.Popen(['diff', '-u', stableOUTPUT + '.ORG',
- stableOUTPUT],
- stdout=f, text=True)
- proc.wait()
+ with process.Popen(['diff', '-u', stableOUTPUT + '.ORG',
+ stableOUTPUT],
+ stdout=f, text=True) as proc:
+ proc.wait()
f.close()
remove(stableOUTPUT + ".ORG")
remove(patch + ".1")
@@ -1608,8 +1608,8 @@ def ApproveOutput (env, TST) :
elif f != thefile and test.match(f):
remove(os.path.join(dir or os.curdir, f + '.rej'))
remove(os.path.join(dir or os.curdir, f + '.orig'))
- proc = process.Popen(patchcmd + ['--forward',
os.path.join(dir or os.curdir, f)], stdin=openutf8(patch), text=True)
- proc.wait()
+ with process.Popen(patchcmd + ['--forward',
os.path.join(dir or os.curdir, f)], stdin=openutf8(patch), text=True) as proc:
+ proc.wait()
if os.path.exists(os.path.join(dir or os.curdir, f
+ '.rej')):
list.append(f)
if len(list) > 0:
@@ -1797,27 +1797,27 @@ def GetBitsAndModsAndThreads(env) :
cmd = splitcommand(env['exe']['MAL_Client'][1])
if procdebug:
print('GetBitsAndModsAndThreads: starting process "%s"
(inpipe, outpipe, errpipe)\n' % '" "'.join(cmd))
- clnt = process.Popen(cmd, stdin=process.PIPE, stdout=process.PIPE,
- stderr=process.PIPE, text=True)
- input = '''\
- c := mdb.modules();
- modsid := algebra.unique(c);
- mods := algebra.projection(modsid,c);
- s := "\\nModules: ";
- sep := "";
- barrier (h:oid,t:str) := iterator.new(mods);
- s := s + sep;
- s := s + "\'";
- s := s + t;
- s := s + "\'";
- sep := ",";
- redo (h:oid,t:str) := iterator.next(mods);
- exit h;
- s := s + "\\n";
- io.printf(s);
- '''
- ##module("NoModule");
- qOut, qErr = clnt.communicate(input=input)
+ with process.Popen(cmd, stdin=process.PIPE, stdout=process.PIPE,
+ stderr=process.PIPE, text=True) as clnt:
+ input = '''\
+ c := mdb.modules();
+ modsid := algebra.unique(c);
+ mods := algebra.projection(modsid,c);
+ s := "\\nModules: ";
+ sep := "";
+ barrier (h:oid,t:str) := iterator.new(mods);
+ s := s + sep;
+ s := s + "\'";
+ s := s + t;
+ s := s + "\'";
+ sep := ",";
+ redo (h:oid,t:str) := iterator.next(mods);
+ exit h;
+ s := s + "\\n";
+ io.printf(s);
+ '''
+ ##module("NoModule");
+ qOut, qErr = clnt.communicate(input=input)
proc.terminate()
sOut = proc.stdout.read()
sErr = proc.stderr.read()
@@ -2498,30 +2498,30 @@ def RunTest(env, TST, BusyPorts, COND, o
if procdebug:
print('RunTest: starting process "%s"\n' % '" "'.join(cmd))
setpgrp = True
- proc = process.Popen(cmd, stdout=process.PIPE,
- stderr=process.PIPE,
- text=True)
- proc.killed = False
- proc.onechild = False
- t = Timer(float(par['TIMEOUT']), killProc, args = [proc])
- try:
- t.start()
- out, err = proc.communicate()
- t.cancel()
- if verbose or quiet:
- if out:
- STDOUT.write(out)
- if err:
- sys.stderr.write(err)
- if procdebug:
- print('RunTest: process exited "%s" (%s)\n' %
- ('" "'.join(cmd), proc.returncode))
- except KeyboardInterrupt:
- t.cancel()
- killProc(proc)
- if procdebug:
- print('RunTest: process killed "%s"\n' % '" "'.join(cmd))
- raise
+ with process.Popen(cmd, stdout=process.PIPE,
+ stderr=process.PIPE,
+ text=True) as proc:
+ proc.killed = False
+ proc.onechild = False
+ t = Timer(float(par['TIMEOUT']), killProc, args = [proc])
+ try:
+ t.start()
+ out, err = proc.communicate()
+ t.cancel()
+ if verbose or quiet:
+ if out:
+ STDOUT.write(out)
+ if err:
+ sys.stderr.write(err)
+ if procdebug:
+ print('RunTest: process exited "%s" (%s)\n' %
+ ('" "'.join(cmd), proc.returncode))
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(proc)
+ if procdebug:
+ print('RunTest: process killed "%s"\n' % '"
"'.join(cmd))
+ raise
timedout = proc.killed
ACCURACYout = ACCURACYout - 1
if ACCURACYout < 0:
@@ -2534,8 +2534,8 @@ def RunTest(env, TST, BusyPorts, COND, o
'-U%s' % par['CONTEXT'],
'%s%s.FILTERED' % (TST, STABLEout),
'%s.test.out.FILTERED' % TST])
- proc = process.Popen(cmd, text=True)
- proc.wait()
+ with process.Popen(cmd, text=True) as proc:
+ proc.wait()
diff_html = openutf8('%s.err.diff.html' % TST,"w")
diff_html.write('<!--MajorDiffs-->\n')
@@ -2567,30 +2567,30 @@ def RunTest(env, TST, BusyPorts, COND, o
if procdebug:
print('RunTest: starting process "%s"\n' % '" "'.join(cmd))
setpgrp = True
- proc = process.Popen(cmd, stdout=process.PIPE,
- stderr=process.PIPE,
- text=True)
- proc.killed = False
- proc.onechild = False
- t = Timer(float(par['TIMEOUT']), killProc, args = [proc])
- try:
- t.start()
- out, err = proc.communicate()
- t.cancel()
- if verbose or quiet:
- if out:
- STDOUT.write(out)
- if err:
- sys.stderr.write(err)
- if procdebug:
- print('RunTest: process exited "%s" (%s)\n' %
- ('" "'.join(cmd), proc.returncode))
- except KeyboardInterrupt:
- t.cancel()
- killProc(proc)
- if procdebug:
- print('RunTest: process killed "%s"\n' % '" "'.join(cmd))
- raise
+ with process.Popen(cmd, stdout=process.PIPE,
+ stderr=process.PIPE,
+ text=True) as proc:
+ proc.killed = False
+ proc.onechild = False
+ t = Timer(float(par['TIMEOUT']), killProc, args = [proc])
+ try:
+ t.start()
+ out, err = proc.communicate()
+ t.cancel()
+ if verbose or quiet:
+ if out:
+ STDOUT.write(out)
+ if err:
+ sys.stderr.write(err)
+ if procdebug:
+ print('RunTest: process exited "%s" (%s)\n' %
+ ('" "'.join(cmd), proc.returncode))
+ except KeyboardInterrupt:
+ t.cancel()
+ killProc(proc)
+ if procdebug:
+ print('RunTest: process killed "%s"\n' % '"
"'.join(cmd))
+ raise
timedout = proc.killed
ACCURACYerr = ACCURACYerr - 1
if ACCURACYerr < 0:
@@ -2603,8 +2603,8 @@ def RunTest(env, TST, BusyPorts, COND, o
'-U%s' % par['CONTEXT'],
'%s%s.FILTERED' % (TST, STABLEerr),
'%s.test.err.FILTERED' % TST])
- proc = process.Popen(cmd, text=True)
- proc.wait()
+ with process.Popen(cmd, text=True) as proc:
+ proc.wait()
FailedOut, FailedErr = CategorizeResult(TST, max(sockerr, errcode),
outmissing, errmissing)
if FailedOut == F_OK and FailedErr == F_OK and testweb:
@@ -2788,10 +2788,10 @@ def Prompt(cmd) :
def getkids():
# return a dictionary with process IDs as key and a list of child
# processes as value
- p = process.Popen(['ps', '-lu', os.getenv('USER')],
- stdout=process.PIPE, stderr=process.PIPE,
- text=True)
- out, err = p.communicate()
+ with process.Popen(['ps', '-lu', os.getenv('USER')],
+ stdout=process.PIPE, stderr=process.PIPE,
+ text=True) as p:
+ out, err = p.communicate()
if err:
return {}
lines = out.split('\n')
@@ -2872,24 +2872,24 @@ def killProc(proc, outfile = None, cmd =
else:
cdb = None
if cdb:
- p = process.Popen([cdb, '-pv', '-p', str(proc.pid),
- '-y',
'%scache*;srv*http://msdl.microsoft.com/download/symbols' % sym, '-lines',
'-c', '~*kP;!locks;q'],
- stdout=process.PIPE, text=True)
- out, err = p.communicate()
+ with process.Popen([cdb, '-pv', '-p', str(proc.pid),
+ '-y',
'%scache*;srv*http://msdl.microsoft.com/download/symbols' % sym, '-lines',
'-c', '~*kP;!locks;q'],
+ stdout=process.PIPE, text=True) as p:
+ out, err = p.communicate()
else:
out = ''
else:
try:
- p = process.Popen(['pstack', str(proc.pid)], stdout=process.PIPE,
- text=True)
- try:
- # pstack (gdb) sometimes hangs when trying to get the
- # stack trace: kill it mercilessly if it does
- t = Timer(60, reallyKill, args = [p])
- t.start()
- except AttributeError:
- t = None
- out, err = p.communicate()
+ with process.Popen(['pstack', str(proc.pid)], stdout=process.PIPE,
+ text=True) as p:
+ try:
+ # pstack (gdb) sometimes hangs when trying to get the
+ # stack trace: kill it mercilessly if it does
+ t = Timer(60, reallyKill, args = [p])
+ t.start()
+ except AttributeError:
+ t = None
+ out, err = p.communicate()
if t is not None:
t.cancel()
except KeyboardInterrupt:
@@ -2910,10 +2910,10 @@ def killProc(proc, outfile = None, cmd =
elif os.name == 'nt':
if procdebug:
print('killProc: starting process "taskkill" "/F" "/T" "/PID"
"%s"\n' % str(proc.pid))
- p = process.Popen(['taskkill','/F','/T','/PID',str(proc.pid)],
- stdout=process.PIPE, stderr=process.PIPE,
- text=True)
- out, err = p.communicate()
+ with process.Popen(['taskkill','/F','/T','/PID',str(proc.pid)],
+ stdout=process.PIPE, stderr=process.PIPE,
+ text=True) as p:
+ out, err = p.communicate()
if procdebug:
print('killProc: process exited "taskkill" "/F" "/T" "/PID" "%s"
(%s)\n' % (str(proc.pid), proc.returncode))
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
