WolfgangZ wrote:
> I'm starting some subprocesses inside a loop. The processes run
> independent and dont need any communication between each other. Due to
> memory issues I need to limit the number of running processes to around
> 10. How can I insert a break into my loop to wait until some processes
> are finished?
>
> Some minimal examplecode:
>
> import subprocess
> for i in range(0,100):
> cmd='ping localhost'
> p=subprocess.Popen(cmd)
> p.wait()
>
Just polling the processes may be good enough:
import random
import subprocess
import sys
import time
from itertools import islice
from functools import partial
PROCESSES = 100
SIMULTANEOUS = 10
def info(*args):
print >> sys.stderr, " ".join(str(a) for a in args)
class Process(object):
def __init__(self, index):
self.index = index
info("starting process #%d" % index)
self.process = subprocess.Popen(["ping", "localhost", "-w", "%d" %
random.randrange(1, 6)])
def __nonzero__(self):
running = self.process.poll() is None
if not running:
# XXX ugly side effect
info("process #%d terminated" % self.index)
return running
def processes():
for i in range(PROCESSES):
yield partial(Process, i)
def main():
starters = processes()
running = [sp() for sp in islice(starters, SIMULTANEOUS)]
while running:
before = len(running)
running = [p for p in running if p]
after = len(running)
if before == after:
info("waiting")
time.sleep(1)
else:
running.extend(sp() for sp in islice(starters, SIMULTANEOUS -
len(running)))
info("that's all, folks")
if __name__ == "__main__":
main()
Peter
--
http://mail.python.org/mailman/listinfo/python-list
