#!/usr/bin/python from mpi4py import MPI import os import re import math import subprocess import time comm = MPI.COMM_WORLD rank = comm.Get_rank() nodes = comm.Get_size() # print(os.path.abspath(os.curdir)) done = set() for f in os.listdir('.'): if re.search('^done_[0-9]+', f): fp = open(f, "r") for x in fp: done.add(int(x)) fp.close() f = open("commands", "r") idx = 0 todo = [] for c in f: if not idx in done: todo.append((idx,c)) idx = idx+1 f.close() start = math.floor(len(todo)/nodes*rank) end = math.floor(len(todo)/nodes*(rank+1)) if(rank == nodes-1): end = len(todo) print("{n:d} commands awaiting execution, {nnode:d} of them in node {rank:d}".format(n=len(todo),nnode=end-start,rank=rank)) time.sleep(1) # to make sure all nodes read the status first before more gets done outfilename = "result_{node:003d}".format(node=rank) donefilename = "done_{node:003d}".format(node=rank) outfile = open(outfilename, "a") donefile = open(donefilename, "a") for i in range(start, end): result = subprocess.call(todo[i][1], stdout=outfile, shell=True) if result == 0: donefile.write(str(todo[i][0]) + '\n') else: print("Command failed: {cmd}".format(cmd=todo[i][1])) outfile.flush() donefile.flush() outfile.close() donefile.close()