WolfgangZ wrote:
I'm starting some subprocesses inside a loop. The processes run
independent and dont need any communication between each other. Due to
memory issues I need to limit the number of running processes to around
10. How can I insert a break into my loop to wait until some processes
are finished?
Some minimal examplecode:
import subprocess
for i in range(0,100):
cmd='ping localhost'
p=subprocess.Popen(cmd)
p.wait()
Just polling the processes may be good enough:
import random
import subprocess
import sys
import time
from itertools import islice
from functools import partial
PROCESSES = 100
SIMULTANEOUS = 10
def info(*args):
print >sys.stderr, " ".join(str(a) for a in args)
class Process(object):
def __init__(self, index):
self.index = index
info("starting process #%d" % index)
self.process = subprocess.Popen(["ping", "localhost", "-w", "%d" %
random.randrange(1, 6)])
def __nonzero__(self):
running = self.process.poll() is None
if not running:
# XXX ugly side effect
info("process #%d terminated" % self.index)
return running
def processes():
for i in range(PROCESSES):
yield partial(Process, i)
def main():
starters = processes()
running = [sp() for sp in islice(starters, SIMULTANEOUS)]
while running:
before = len(running)
running = [p for p in running if p]
after = len(running)
if before == after:
info("waiting")
time.sleep(1)
else:
running.extend(sp() for sp in islice(starters, SIMULTANEOUS -
len(running)))
info("that's all, folks")
if __name__ == "__main__":
main()
Peter
