1
2 """Things that are needed for convenient parallel Execution"""
3
4 from PyFoam.Basics.Utilities import Utilities
5 from PyFoam.FoamInformation import foamMPI,oldAppConvention
6 from PyFoam.Error import error,warning,debug
7 from PyFoam import configuration as config
8
9 from os import path,environ,system
10
11 import subprocess
12
14 """Wrapper class for starting an stopping a LAM-Machine"""
15 - def __init__(self,machines=None,nr=None):
16 """@param machines: Name of the file with the machine information
17 @param nr: Number of processes"""
18
19 Utilities.__init__(self)
20
21 self.stop()
22
23 if machines=="":
24 machines=None
25
26 if machines==None and foamMPI()=="LAM":
27 error("Machinefile must be specified for LAM")
28
29 if machines==None and nr==None:
30 error("Either machinefile or Nr of CPUs must be specified for MPI type",foamMPI())
31
32 self.mFile=machines
33 self.procNr=nr
34
35 self.boot()
36 if not self.machineOK():
37 error("Error: LAM was not started")
38
40 """Check whether the LAM machine was properly booted"""
41 if self.running:
42 if(foamMPI()=="LAM"):
43 if self.cpuNr()<=0:
44 self.running=False
45
46 return self.running
47
49 """Stops a LAM-machine (if one is running)"""
50 self.running=False
51 if(foamMPI()=="LAM"):
52 self.execute("lamhalt -v")
53
55 """Boots a LAM-machine using the machine-file"""
56 if foamMPI()=="LAM":
57 warning("LAM is untested. Any Feedback most welcome")
58 self.execute("lamboot -s -v "+self.mFile)
59 self.running=True
60 elif foamMPI().find("OPENMPI")>=0:
61 self.running=True
62 else:
63 error(" Unknown or missing MPI-Implementation: "+foamMPI())
64
66 if(foamMPI()=="LAM"):
67 if self.running:
68 lines=self.execute("lamnodes")
69 nr=0
70 for l in lines:
71 tmp=l.split(':')
72 if len(tmp)>1:
73 nr+=int(tmp[1])
74 return nr
75 else:
76 return -1
77 elif foamMPI().find("OPENMPI")>=0:
78 if self.mFile:
79 f=open(self.mFile)
80 l=[i.strip() for i in f.readlines()]
81 f.close()
82 nr=0
83 for m in l:
84 tmp=m.split()
85 if len(tmp)==1:
86 nr+=1
87 elif len(tmp)==0:
88 pass
89 else:
90 error("Machinefile not valid (I think): more than one element in one line:"+str(tmp)+"\nPropably you wrote a line in the form 'node1 cpu=2', but I only understand 'node1\\nnode1'")
91
92 if self.procNr==None:
93 return nr
94 else:
95 return min(nr,self.procNr)
96
97 elif self.procNr:
98 return self.procNr
99 else:
100 error("Can't determine Nr of CPUs without machinefile")
101
103 """Builds a list with a working mpirun command (for that MPI-Implementation)
104 @param argv: the original arguments that are to be wrapped
105 @param expandApplication: Expand the
106 @return: list with the correct mpirun-command"""
107
108 nr=str(self.cpuNr())
109 mpirun=[config().get("MPI","run_"+foamMPI(),default="mpirun")]
110 mpiRunPath=self.which(" ".join(mpirun))
111 if not mpiRunPath:
112 error("Could not find a path for",mpirun,
113 "Check configuration variable","run_"+foamMPI(),
114 "in section 'MPI'")
115 mpirun+=eval(config().get("MPI","options_"+foamMPI()+"_pre",default="[]"))
116
117 if(foamMPI()=="LAM"):
118 mpirun+=["-np",nr]
119 elif foamMPI().find("OPENMPI")>=0:
120 nr=[]
121 if "MPI_ARCH_PATH" in environ and config().getboolean("MPI","OpenMPI_add_prefix"):
122 nr+=["--prefix",environ["MPI_ARCH_PATH"]]
123 if self.procNr!=None:
124 nr+=["--n",str(self.procNr)]
125 machine=[]
126 if self.mFile!=None:
127 machine=["--machinefile",self.mFile]
128 if config().getdebug("ParallelExecution"):
129 debug("Start of",self.mFile)
130 debug("\n"+open(self.mFile).read())
131 debug("End of",self.mFile)
132 mpirun+=machine+nr
133 else:
134 error(" Unknown or missing MPI-Implementation for mpirun: "+foamMPI())
135
136 mpirun+=eval(config().get("MPI","options_"+foamMPI()+"_post",default="[]"))
137
138 progname=argv[0]
139 if expandApplication:
140
141 progname=self.which(progname)
142 if progname:
143 progname=argv[0]
144 warning("which can not find a match for",progname,". Hoping for the best")
145
146 if oldAppConvention():
147 mpirun+=[progname]+argv[1:3]+["-parallel"]+argv[3:]
148 else:
149 mpirun+=[progname]+argv[1:]+["-parallel"]
150
151 if config().getdebug("ParallelExecution"):
152 debug("MPI:",foamMPI())
153 debug("Arguments:",mpirun)
154 system("which mpirun")
155 system("which rsh")
156 debug("Environment",environ)
157 for a in mpirun:
158 if a in environ:
159 debug("Transfering variable",a,"with value",environ[a])
160
161 return mpirun
162
164 """Write the parameter-File for a metis decomposition
165 @param sDir: Solution directory
166 @type sDir: PyFoam.RunDictionary.SolutionDirectory"""
167
168 params="method metis;\n"
169
170 self.writeDecomposition(sDir,params)
171
173 """Write the parameter-File for a metis decomposition
174 @param sDir: Solution directory
175 @type sDir: PyFoam.RunDictionary.SolutionDirectory
176 @param direction: direction in which to decompose (0=x, 1=y, 2=z)"""
177
178 params ="method simple;\n"
179 params+="\nsimpleCoeffs\n{\n\t n \t ("
180 if direction==0:
181 params+=str(self.cpuNr())+" "
182 else:
183 params+="1 "
184 if direction==1:
185 params+=str(self.cpuNr())+" "
186 else:
187 params+="1 "
188 if direction==2:
189 params+=str(self.cpuNr())
190 else:
191 params+="1"
192 params+=");\n\t delta \t 0.001;\n}\n"
193
194 self.writeDecomposition(sDir,params)
195
197 """Write parameter file for a decomposition
198 @param par:Parameters specific for that kind of decomposition
199 @type par:str
200 @param sDir: Solution directory
201 @type sDir: PyFoam.RunDictionary.SolutionDirectory"""
202
203 f=open(path.join(sDir.systemDir(),"decomposeParDict"),"w")
204 self.writeDictionaryHeader(f)
205 f.write("// * * * * * * * * * //\n\n")
206 f.write("numberOfSubdomains "+str(self.cpuNr())+";\n\n")
207 f.write(par)
208 f.write("\n\n// * * * * * * * * * //")
209 f.close()
210
211
212