Changeset: b735da1fd959 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB/rev/b735da1fd959 Modified Files: sql/backends/monet5/vaults/csv/CMakeLists.txt testing/Mtest.py.in Branch: default Log Message:
Merge with Aug2024 branch. diffs (truncated from 356 to 300 lines): diff --git a/clients/mapiclient/curl-stream.h b/clients/mapiclient/curl-stream.h --- a/clients/mapiclient/curl-stream.h +++ b/clients/mapiclient/curl-stream.h @@ -1,3 +1,15 @@ +/* + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 2024, 2025 MonetDB Foundation; + * Copyright August 2008 - 2023 MonetDB B.V.; + * Copyright 1997 - July 2008 CWI. + */ + #include <curl/curl.h> #ifndef CURL_WRITEFUNC_ERROR diff --git a/clients/mapiclient/iconv-stream.h b/clients/mapiclient/iconv-stream.h --- a/clients/mapiclient/iconv-stream.h +++ b/clients/mapiclient/iconv-stream.h @@ -1,3 +1,15 @@ +/* + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 2024, 2025 MonetDB Foundation; + * Copyright August 2008 - 2023 MonetDB B.V.; + * Copyright 1997 - July 2008 CWI. + */ + #include <iconv.h> struct ic_priv_t { diff --git a/testing/Mtest.py.in b/testing/Mtest.py.in --- a/testing/Mtest.py.in +++ b/testing/Mtest.py.in @@ -1357,7 +1357,7 @@ def PerformDir(env, testdir, testlist, t else: vaultopt = [] if not oneserver: - pSrvr = ServerClass(splitcommand(env['exe']['mserver5'][1] +['--dbpath=%s' % LogDBdir] + vaultopt + mserver5_opts), open(os.devnull, 'w'), open(os.devnull, 'w'), par['TIMEOUT'], os.path.join(LogDBdir, '.started'), dbg=env.get('DBG')) + pSrvr = ServerClass(splitcommand(env['exe']['mserver5'][1] +[f'--dbpath={LogDBdir}'] + vaultopt + mserver5_opts), open(os.devnull, 'w'), open(os.devnull, 'w'), par['TIMEOUT'], os.path.join(LogDBdir, '.started'), TSTDB, dbg=env.get('DBG')) pSrvr.LaunchIt() pSrvr.terminate() if not os.path.exists(TSTTRGDIR): @@ -1405,6 +1405,7 @@ def PerformDir(env, testdir, testlist, t openutf8(os.path.join(TSTTRGDIR, 'SingleServer.err'), 'a'), 0, pollfile, + TSTDB, inmem=inmem, dbg=env.get('DBG')) os.chdir(TSTTRGDIR) @@ -1419,9 +1420,14 @@ def PerformDir(env, testdir, testlist, t hostname=HOST, port=int(pSrvr.port), database=TSTDB, - autocommit=True) + autocommit=True, + connect_timeout=1.0) except KeyboardInterrupt: raise + except ConnectionRefusedError: + print('\nConnection refused.\n') + FdOut = FdErr = F_FAIL + break except: pass else: @@ -2633,16 +2639,7 @@ def reallyKill(proc): # not called on Windows killchildren(proc.pid, getkids()) -def killProc(proc, outfile = None, cmd = None): - if type(cmd) is type([]): - cmd = ' '.join(cmd) - if procdebug: - print('timeout for process %d (%s)' % (proc.pid, cmd)) - if outfile is not None and cmd is not None: - try: - outfile.write('\n!Mtimeout: Timeout: %s\n' % cmd) - except (ValueError, IOError): - print('cannot write timeout message ' + cmd) +def stacktrace(proc, outfile): if os.name == "nt": sym = '' if os.path.exists(r'c:\Program Files\Debugging Tools for Windows (x64)\cdb.exe'): @@ -2657,14 +2654,22 @@ def killProc(proc, outfile = None, cmd = cdb = None if cdb: with process.Popen([cdb, '-pv', '-p', str(proc.pid), - '-y', '%scache*;srv*http://msdl.microsoft.com/download/symbols' % sym, '-lines', '-c', '~*kP;!locks;q'], + '-y', f'{sym}cache*;srv*http://msdl.microsoft.com/download/symbols', '-lines', '-c', '~*kP;!locks;q'], stdout=process.PIPE, text=True) as p: out, err = p.communicate() else: out = '' + elif platform.system() == 'Darwin': + try: + with process.Popen(['lldb', '--attach-pid', str(proc.pid), '--batch', '--one-line', 'bt all'], stdout=process.PIPE, text=True) as p: + out, err = p.communicate() + except KeyboardInterrupt: + raise + except: + out = '' else: try: - with process.Popen(['gdb', '-p', str(proc.pid), '-batch', '-ex', 'thread apply all bt full', '-ex', 'call dump_threads()'], stdout=process.PIPE, + with process.Popen(['gdb', '-p', str(proc.pid), '-batch', '-ex', 'thread apply all bt full'], stdout=process.PIPE, text=True) as p: try: # gdb sometimes hangs when trying to get the stack @@ -2682,15 +2687,32 @@ def killProc(proc, outfile = None, cmd = out = '' if outfile is not None and out: try: - outfile.write('\n%s\n' % out) + outfile.write(f'\n{out}\n') except (ValueError, IOError): print('cannot write stack trace') print(out) + +def killProc(proc, outfile=None, cmd=None, psrvr=None): + if type(cmd) is type([]): + cmd = ' '.join(cmd) + if procdebug: + print('timeout for process %d (%s)' % (proc.pid, cmd)) + if outfile is not None and cmd is not None: + try: + outfile.write('\n!Mtimeout: Timeout: %s\n' % cmd) + except (ValueError, IOError): + print('cannot write timeout message ' + cmd) + stacktrace(proc, outfile) proc.killed = True if proc.onechild: - if procdebug: - print('killProc: calling proc.kill() on PID %d' % proc.pid) - proc.kill() + if psrvr is not None and psrvr.proc is proc: + if procdebug: + print(f'killProc: calling psrvr.terminate() on PID {proc.pid}') + psrvr.terminate() + else: + if procdebug: + print(f'killProc: calling proc.kill() on PID {proc.pid}') + proc.kill() elif os.name == 'nt': if procdebug: print('killProc: starting process "taskkill" "/F" "/T" "/PID" "%s"\n' % str(proc.pid)) @@ -2705,7 +2727,7 @@ def killProc(proc, outfile = None, cmd = killchildren(proc.pid) class ServerClass: - def __init__(self, cmd, TestOut, TestErr, TimeOut, pollfile, inmem=False, dbg=None): + def __init__(self, cmd, TestOut, TestErr, TimeOut, pollfile, dbname, inmem=False, dbg=None): self.proc = None self.timer = None self.outfile = TestOut @@ -2717,13 +2739,16 @@ class ServerClass: self.pollfile = pollfile self.inmem = inmem self.dbg = dbg + self.dbname = dbname + self.running = None + self.lock = threading.Lock() def poll(self): return self.proc.poll() def terminate(self): self.timer.cancel() - t = Timer(60, killProc, args = [self.proc, self.errfile, self.cmd]) + t = Timer(60, killProc, args=[self.proc, self.errfile, self.cmd, self]) t.start() if os.name == 'nt': self.proc.send_signal(signal.CTRL_BREAK_EVENT) @@ -2737,11 +2762,53 @@ class ServerClass: self.outfile.close() self.errfile.close() + def start(self, timeout): + self.running = Timer(timeout, self.stopsessions, []) + self.running.start() + + def stop(self): + if self.lock.acquire(): + if self.running: + self.running.cancel() + self.running = None + self.lock.release() + def sendusr1(self): if os.name != 'nt': + if procdebug: + print(f'sendusr1: sending USR1 signal to {self.proc.pid}') self.proc.send_signal(signal.SIGUSR1) time.sleep(1) + def stacktrace(self): + stacktrace(self.proc, self.outfile) + self.sendusr1() + + def stopsessions(self): + if self.lock.acquire(blocking=False): + if self.running is not None: + self.stacktrace() + try: + dbh = pymonetdb.connect(username='monetdb', + password='monetdb', + hostname=HOST, + port=int(self.port), + database=self.dbname, + connect_timeout=1.0) + crs = dbh.cursor() + crs.execute('select sessionid from sys.sessions() where sessionid <> sys.current_sessionid()') + ids = crs.fetchall() + for x in ids: + if procdebug: + print(f'stopping session {x[0]}') + crs.execute(f'call sys.stopsession({x[0]})') + if procdebug and not ids: + print('no sessions to stop') + except: + pass + self.running = None + self.lock.release() + def LaunchIt(self): global setpgrp @@ -2781,11 +2848,10 @@ class ServerClass: proc.stderr = process._BufferedPipe(proc.stderr) proc.killed = False proc.onechild = True - t = Timer(self.timeout, killProc, args = [proc, self.errfile, self.cmd]) + self.timer = Timer(self.timeout, killProc, args=[proc, self.errfile, self.cmd, self]) if self.timeout > 0: - t.start() + self.timer.start() self.proc = proc - self.timer = t port = None self.port = None @@ -2795,7 +2861,7 @@ class ServerClass: if proc.returncode is not None: # exited proc.wait() - t.cancel() + self.timer.cancel() return if os.path.exists(self.pollfile): break @@ -2822,14 +2888,14 @@ class ServerClass: else: proc.terminate() proc.wait() - t.cancel() + self.timer.cancel() return self.started = True self.port = port ### LaunchIt(cmd, TestIn, TestOut, TestErr, TimeOut, pollfile, port) # -def RunIt(cmd, onechild, TestIn, TestOut, TestErr, TimeOut) : +def RunIt(cmd, onechild, TestIn, TestOut, TestErr, TimeOut, pSrvr) : global setpgrp if type(TestIn) is type(''): TestInput = TestIn @@ -2845,7 +2911,7 @@ def RunIt(cmd, onechild, TestIn, TestOut stderr=TestErr, text=True) as proc: proc.killed = False proc.onechild = onechild - t = Timer(TimeOut, killProc, args = [proc, TestErr, cmd]) + t = Timer(TimeOut, killProc, args = [proc, TestErr, cmd, pSrvr]) try: t.start() # since both stdout and stderr are redirected to files, @@ -2982,7 +3048,7 @@ def DoIt(env, SERVER, CALL, TST, EXT, Te Srvr.extend(openutf8(TST + '.options5').read().split()) Srvr.extend(mserver5_opts) Srvr.extend(options) - pSrvr = ServerClass(Srvr, SrvrOut, SrvrErr, TIMEOUT, os.path.join(dbpath, '.started'), dbg=env.get('DBG')) + pSrvr = ServerClass(Srvr, SrvrOut, SrvrErr, TIMEOUT, os.path.join(dbpath, '.started'), TSTDB, dbg=env.get('DBG')) pSrvr.LaunchIt() if pSrvr.port is None: print('\nFailed to start server.\n') @@ -2990,8 +3056,9 @@ def DoIt(env, SERVER, CALL, TST, EXT, Te os.environ['MAPIPORT'] = env['MAPIPORT'] = pSrvr.port SetExecEnv(exe,pSrvr.port,verbosity > 1) else: - PSRVR.timer.settimeout(TIMEOUT) - PSRVR.timer.start() + PSRVR.start(TIMEOUT) _______________________________________________ checkin-list mailing list -- checkin-list@monetdb.org To unsubscribe send an email to checkin-list-le...@monetdb.org
