1
2 """
3 Application class that implements pyFoamComparator
4 """
5
6 import sys
7 import re
8 import string
9 from xml.dom.minidom import parse
10 import xml.dom
11 from os import path,environ
12
13 from PyFoam.Error import error
14 from PyFoam.Basics.Utilities import execute
15 from PyFoam.Execution.AnalyzedRunner import AnalyzedRunner
16 from PyFoam.Execution.ConvergenceRunner import ConvergenceRunner
17 from PyFoam.Execution.BasicRunner import BasicRunner
18 from PyFoam.Execution.UtilityRunner import UtilityRunner
19 from PyFoam.Execution.ParallelExecution import LAMMachine
20 from PyFoam.RunDictionary.ParsedParameterFile import ParsedParameterFile
21 from PyFoam.LogAnalysis.BoundingLogAnalyzer import BoundingLogAnalyzer
22 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
23 from PyFoam.Basics.CSVCollection import CSVCollection
24
25 from PyFoamApplication import PyFoamApplication
26 from PyFoam.FoamInformation import changeFoamVersion,injectVariables
27
28 from Decomposer import Decomposer
29
32 description="""
33 Reads an XML-file that specifies a base case and a parameter-variation
34 and executes all the variations of that case
35 """
36
37 PyFoamApplication.__init__(self,args=args,description=description,usage="%prog [options] <xmlfile>",nr=1,interspersed=True)
38
40 self.parser.add_option("--test",
41 action="store_true",
42 default=False,
43 dest="test",
44 help="Only does the preparation steps but does not execute the actual solver an the end")
45
46 self.parser.add_option("--removeOld",
47 action="store_true",
48 default=False,
49 dest="removeOld",
50 help="Remove the directories from an old run without asking")
51
52 self.parser.add_option("--purge",
53 action="store_true",
54 default=False,
55 dest="purge",
56 help="Remove the case directories after evaluating")
57
58 self.parser.add_option("--no-purge",
59 action="store_true",
60 default=False,
61 dest="nopurge",
62 help="Don't remove the case directories after evaluating")
63
64 self.parser.add_option("--steady",
65 action="store_true",
66 default=False,
67 dest="steady",
68 help="Only runs the solver until convergence")
69
70 self.parser.add_option("--showDictionary",
71 action="store_true",
72 default=False,
73 dest="showDict"
74 ,help="Shows the parameter-dictionary after the running of the solver")
75
76 self.parser.add_option("--foamVersion",
77 dest="foamVersion",
78 default=None,
79 help="Change the OpenFOAM-version that is to be used")
80
81 self.parser.add_option("--no-server",
82 dest="server",
83 default=True,
84 action="store_false",
85 help="Don't start the process-control-server")
86
88 if self.opts.foamVersion!=None:
89 changeFoamVersion(self.opts.foamVersion)
90
91 fName=self.parser.getArgs()[0]
92
93 dom=parse(fName)
94 doc=dom.documentElement
95
96 if doc.tagName!='comparator':
97 error("Wrong root-element",doc.tagName,"Expected: 'comparator'")
98
99 self.data=ComparatorData(doc)
100
101 purge=False
102 if doc.hasAttribute('purge'):
103 purge=eval(doc.getAttribute('purge'))
104 if self.opts.purge:
105 purge=self.opts.purge
106 if self.opts.nopurge:
107 purge=False
108
109 steady=False
110 if doc.hasAttribute('steady'):
111 steady=eval(doc.getAttribute('steady'))
112 if self.opts.steady:
113 purge=self.opts.steady
114
115 print " Parameters read OK "
116 print
117
118 aLog=open(self.data.id+".overview","w")
119 csv=CSVCollection(self.data.id+".csv")
120
121 rDir=self.data.id+".results"
122 execute("rm -rf "+rDir)
123 execute("mkdir "+rDir)
124
125 calculated=0
126 format="%%0%dd" % len(str(len(self.data)))
127
128 for i in range(len(self.data)):
129 runID=(format % i)
130 print >>aLog,runID,
131 csv["ID"]=runID
132
133 use,para=self.data[i]
134 para["template"]=self.data.template
135 para["extension"]=self.data.extension
136 para["id"]=self.data.id
137
138 if use:
139 calculated+=1
140
141 print "Executing Variation",i+1,"of",len(self.data),
142 if calculated!=i+1:
143 print "(",calculated,"actually calculated)"
144 else:
145 print
146
147 print "Parameters:",
148 for k,v in para.iteritems():
149 print "%s='%s' " % (k,v),
150 if v.find(" ")>=0 or v.find("\t")>=0:
151 v="'"+v+"'"
152 print >>aLog,v,
153 csv[k]=v
154
155 print
156
157 if not use:
158 print "Skipping because not all conditions are satisfied"
159 csv.clear()
160 print
161 continue
162
163 cName=("%s."+format) % (self.data.id, i)
164 log=open(cName+".log","w")
165
166 para["case"]=cName
167 print "Case-directory:",cName
168 para["results"]=path.join(rDir,runID)
169 print "Results directory:",para["results"]
170 execute("mkdir "+para["results"])
171
172 if path.exists(cName):
173 if self.opts.removeOld:
174 print " Removing old case-directory"
175 execute("rm -r "+cName)
176 else:
177 error("Case-directory",cName,"exists")
178
179 print " copying template"
180 out=execute("cp -r "+self.data.template+" "+cName)
181 print >>log,"---- Copying"
182 for l in out:
183 print >>log,l,
184
185 print " preparing"
186 ok,erg=self.data.prep.execute(para,log)
187 print >>aLog,ok,
188 csv["prepare OK"]=ok
189
190 for i in range(len(erg)):
191 print >>aLog,erg[i],
192 csv["Prepare %02d" % i]=erg[i]
193
194 aLog.flush()
195
196 if self.opts.test:
197 print " Skipping execution"
198 else:
199 print " running the solver"
200 sys.stdout.flush()
201
202 if steady:
203 runnerClass=ConvergenceRunner
204 else:
205 runnerClass=AnalyzedRunner
206
207 run=runnerClass(BoundingLogAnalyzer(doTimelines=True,progress=True),
208 argv=[self.data.solver,".",cName],
209 silent=True,
210 lam=Command.parallel,
211 server=self.opts.server)
212
213 run.start()
214 ok=run.runOK()
215 if ok:
216 print " executed OK"
217 else:
218 print " fatal error"
219
220 for aName in run.listAnalyzers():
221 a=run.getAnalyzer(aName)
222 if 'titles' in dir(a):
223 for tit in a.lines.getValueNames():
224 t,v=a.getTimeline(tit)
225 if len(v)>0:
226 para["result_"+aName+"_"+tit]=v[-1]
227
228 print >>aLog,run.runOK(),run.lastTime(),run.run.wallTime(),
229 csv["Run OK"]=run.runOK()
230 csv["End Time"]=run.lastTime()
231 csv["Wall Time"]=run.run.wallTime()
232 csv["Wall Time (Foam)"]=run.totalClockTime()
233 csv["CPU Time"]=run.totalCpuTime()
234 csv["Wall Time First Step"]=run.firstClockTime()
235 csv["CPU Time First Step"]=run.firstCpuTime()
236
237 para["endTime"]=run.lastTime()
238 para["runlog"]=run.logFile
239
240 if self.opts.showDict:
241 print para
242
243 print " evaluating results"
244
245 ok,erg=self.data.post.execute(para,log)
246
247 if Command.parallel!=None:
248 print " Stoping LAM"
249 Command.parallel.stop()
250 Command.parallel=None
251
252 if ok:
253 print " Evaluation OK",
254 else:
255 print " Evaluation failed",
256
257 if len(erg)>0:
258 print ":",erg,
259 print
260
261 print >>aLog,ok,
262 for i in range(len(erg)):
263 print >>aLog,erg[i],
264 csv["Post %02d" % i]=erg[i]
265
266 if purge:
267 print " removing the case-directory"
268 out=execute("rm -r "+cName)
269 print >>log,"---- Removing"
270 for l in out:
271 print >>log,l,
272
273 log.close()
274 print
275 print >>aLog
276 aLog.flush()
277 csv.write()
278
279 aLog.close()
280
282 """ The object that holds the actual data"""
283
285 """
286 @param doc: the parsed XML-data from which the object is constructed
287 """
288 self.name=doc.getAttribute("name")
289 if self.name=="":
290 error("No name for 'comparator' given")
291
292 base=doc.getElementsByTagName("base")
293 if base.length!=1:
294 error("One 'base'-element needed. Found",base.length)
295 self.__parseBase(base[0])
296
297 self.vList=[]
298 for v in doc.getElementsByTagName("variation"):
299 self.vList.append(Variation(v))
300
302 """@param e: The 'base'-element"""
303
304 self.template=path.expandvars(e.getAttribute("template"))
305 if self.template=="":
306 error("No template is given")
307 if not path.exists(self.template):
308 error("Template",self.template,"does not exist")
309 self.id=path.basename(self.template)
310 if e.hasAttribute('extension'):
311 self.extension=e.getAttribute('extension')
312 self.id+="."+self.extension
313 else:
314 self.extension=""
315 self.solver=e.getAttribute("solver")
316 if self.solver=="":
317 error("No solver is given")
318 prep=e.getElementsByTagName("preparation")
319 if prep.length!=1:
320 error("One 'preparation'-element needed. Found",prep.length)
321 self.prep=PreparationChain(prep[0])
322 post=e.getElementsByTagName("evaluation")
323 if post.length!=1:
324 error("One 'evaluation'-element needed. Found",post.length)
325 self.post=EvaluationChain(post[0])
326
328 """@return: The total number of variations"""
329 if len(self.vList)==0:
330 return 0
331 else:
332 nr=1l
333 for v in self.vList:
334 nr*=len(v)
335 return nr
336
338 """@param nr: Number of the variation
339 @return: dictionary with the variation"""
340 if nr>=len(self):
341 error("Index",nr,"of variation out of bounds: [0,",len(self)-1,"]")
342 result={}
343 tmp=nr
344 conditions=[]
345 for v in self.vList:
346 if (tmp % len(v))!=0:
347 conditions.append(v.condition)
348
349 k,val=v[tmp % len(v)]
350 result[k]=val
351 tmp/=len(v)
352
353 assert tmp==0
354
355 use=True
356 for c in conditions:
357 cond=replaceValues(c,result)
358 use=use and eval(cond)
359
360 return use,result
361
363 """Abstract base class for a number of commands"""
365 """@param c: XML-Subtree that represents the chain"""
366 self.commands=[]
367 for e in c.childNodes:
368 if e.nodeType!=xml.dom.Node.ELEMENT_NODE:
369 continue
370 if not e.tagName in self.table.keys():
371 error("Tagname",e.tagName,"not in table of valid tags",self.table.keys())
372 self.commands.append(self.table[e.tagName](e))
373
375 """Executes the chain
376 @param para:A dictionary with the parameters
377 @param log: Logfile to write to"""
378
379 result=[]
380 status=True
381 for c in self.commands:
382
383 if c.doIt(para):
384 ok,erg=c.execute(para,log)
385 else:
386 ok,erg=True,[]
387
388 status=ok and status
389 if erg!=None:
390 if type(erg)==list:
391 result+=erg
392 else:
393 result.append(erg)
394
395 return status,result
396
398 """Checks whether there is an object of a specific type"""
399
400 for o in self.commands:
401 if type(o)==typ:
402 return True
403
404 return False
405
407 """Chain of Preparation commands"""
409 self.table={"genericcommand":GenericCommand,
410 "derived":DerivedCommand,
411 "foamcommand":FoamCommand,
412 "foamutility":FoamUtilityCommand,
413 "initial":InitialCommand,
414 "dictwrite":DictWriteCommand,
415 "setdictionary":SetDictionaryCommand,
416 "decompose":DecomposeCommand,
417 "foamversion":FoamVersionCommand,
418 "changeenvironment":ChangeEnvironmentCommand,
419 "setenv":SetEnvironmentCommand,
420 "boundary":BoundaryCommand}
421 CommandChain.__init__(self,c)
422
424 """Chain of evaluation commands"""
426 self.table={"genericcommand":GenericCommand,
427 "derived":DerivedCommand,
428 "foamutility":FoamUtilityCommand,
429 "foamcommand":FoamCommand,
430 "dictionary":DictionaryCommand,
431 "reconstruct":ReconstructCommand,
432 "lastresult":LastResultCommand,
433 "changeenvironment":ChangeEnvironmentCommand,
434 "setenv":SetEnvironmentCommand,
435 "copylog":CopyLogCommand}
436 CommandChain.__init__(self,c)
437
439 result=e.getAttribute(name)
440 if result=="":
441 if default==None:
442 error("Missing attribute",name,"in element",e.tagName)
443 else:
444 return default
445 return result
446
448 """Replaces all strings enclosed by $$ with the parameters
449 @param orig: the original string
450 @param para: dictionary with the parameters"""
451
452 exp=re.compile("\$[^$]*\$")
453 tmp=orig
454
455 m=exp.search(tmp)
456 while m:
457 a,e=m.span()
458 pre=tmp[0:a]
459 post=tmp[e:]
460 mid=tmp[a+1:e-1]
461
462 if not mid in para.keys():
463 error("Key",mid,"not existing in keys",para.keys())
464
465 tmp=pre+para[mid]+post
466
467 m=exp.search(tmp)
468
469 return tmp
470
472
473 parallel=None
474
475 """Abstract base class of all commands"""
478
479 - def doIt(self,para):
483
485 """@param vals: Dictionary with the keywords
486 @return: A boolean whether it completed successfully and a list with results (None if no results are generated)"""
487 error("Execute not implemented for",type(self))
488
490 """Executes a shell command"""
494
496 cmd=replaceValues(self.command,para)
497 print " Executing ",cmd,
498 sys.stdout.flush()
499 out=execute(cmd)
500
501 if len(out)>0:
502 print " -->",len(out),"lines output"
503 for l in out:
504 print >>log,"---- Command:",cmd
505 print >>log,l,
506 else:
507 print
508
509 return True,None
510
512 """Derives an additional value"""
517
519 tmp=replaceValues(self.expression,para)
520 try:
521 val=eval(tmp)
522 except SyntaxError:
523 error("Syntax error in",tmp)
524 print " Setting",self.name,"to",val
525 para[self.name]=str(val)
526
527 return True,None
528
530 """Returns values from the chains dictionaries"""
534
536 if para.has_key(self.key):
537 return True,para[self.key]
538 else:
539 print "-----> ",self.key,"not in set of valid keys",para.keys()
540 print >>log,self.key,"not in set of valid keys of dictionary",para
541 return False,None
542
544 """Sets value in the chains dictionaries"""
549
551 para[self.key]=self.value
552 return True,None
553
555 """Changes the used OpenFOAM-version"""
559
565
567 """Sets an environment variable"""
572
574 val=replaceValues(self.val,para)
575 var=replaceValues(self.var,para)
576 print " Setting variable",var,"to",val
577 environ[var]=val
578
579 return True,None
580
582 """Changes Environment variables by executing a script-file"""
586
588 script=replaceValues(self.script,para)
589 print " Changing environment variables by executing",script
590 injectVariables(script)
591
592 return True,None
593
595 """Decomposes a case and generates the LAM"""
601
615
617 """Reconstructs a case and deleted the LAM"""
619 Command.__init__(self,c)
620 self.onlyLatest=False
621 if c.hasAttribute('onlyLatest'):
622 self.onlyLatest=eval(c.getAttribute('onlyLatest'))
623
638
640 """Executes a OpenFOAM-utility"""
645
658
660 """Executes a OpenFOAM-utility and extracts information"""
664
666 argv=[self.utility,".",para['case']]+self.options.split()
667 print " Executing and analyzing",string.join(argv),
668 sys.stdout.flush()
669 run=UtilityRunner(argv,silent=True,lam=Command.parallel,logname=string.join(argv,"_"))
670 run.add("data",self.regexp)
671 run.start()
672 data=run.analyzer.getData("data")
673 result=None
674 if data!=None:
675 result=[]
676 for a in data:
677 result.append(a)
678 if result==None:
679 print "no data",
680 else:
681 print result,
682
683 if run.runOK():
684 print
685 else:
686 print "---> there was a problem"
687
688 return run.runOK(),result
689
691 """Common class for commands that operate on dictionaries"""
695
697 f=replaceValues(self.filename,para)
698 v=replaceValues(self.value,para)
699 s=replaceValues(self.subexpression,para)
700 k=replaceValues(self.key,para)
701
702 try:
703 dictFile=ParsedParameterFile(f,backup=True)
704 val=dictFile[k]
705 except KeyError:
706 self.error("Key: ",k,"not existing in File",f)
707 except IOError,e:
708 self.error("Problem with file",k,":",e)
709
710 try:
711 exec "dictFile[k]"+s+"=v"
712 except Exception,e:
713 error("Problem with subexpression:",sys.exc_info()[0],":",e)
714
715 dictFile.writeFile()
716
717 return True,None
718
720 """Common class for commands that set values on fields"""
722 SetterCommand.__init__(self,c)
723 self.field=c.getAttribute("field")
724 self.filename=path.join("$case$","0",self.field)
725
727 """Sets an initial condition"""
732
736
738 """Sets a boundary condition"""
740 FieldSetterCommand.__init__(self,c)
741 self.patch=c.getAttribute("patch")
742 self.key="boundaryField"
743 self.subexpression="['"+self.patch+"']"
744 self.element=c.getAttribute("element")
745 if self.element=="":
746 self.element="value"
747 self.subexpression+="['"+self.element+"']"
748
752
754 """Writes a value to a dictionary"""
756 SetterCommand.__init__(self,c)
757 self.dir=c.getAttribute("directory")
758 self.dict=c.getAttribute("dictionary")
759 self.filename=path.join("$case$",self.dir,self.dict)
760 self.key=c.getAttribute("key")
761 self.subexpression=c.getAttribute("subexpression")
762 self.value=c.getAttribute("value")
763
767
769 """Copies the result of the last time-step to the resultsd directory"""
772
779
781 """Copies the log file to the results"""
784
786 print " Copy logfile"
787 execute("cp "+para["runlog"]+" "+para["results"])
788 return True,None
789
791 """Represents one variation"""
792
794 """@param e: the XML-data from which it is created"""
795
796 self.name=e.getAttribute("name")
797 if self.name=="":
798 error("No name for 'variation' given")
799 self.key=e.getAttribute("key")
800 if self.key=="":
801 error("No key for 'variation'",self.name," given")
802 self.condition=e.getAttribute("condition")
803 if self.condition=="":
804 self.condition="True"
805 self.values=[]
806 for v in e.getElementsByTagName("value"):
807 self.values.append(str(v.firstChild.data))
808
810 return "Variation "+self.name+" varies "+self.key+" over "+str(self.values)
811
813 """@return: number of values"""
814 return len(self.values)
815
817 return self.key,self.values[key]
818