1
2 """
3 Application class that implements pyFoamComparator
4 """
5
6 import sys
7 import re
8 import string
9 from xml.dom.minidom import parse
10 import xml.dom
11 from os import path,environ,mkdir
12 from optparse import OptionGroup
13
14 from PyFoam.Error import error
15 from PyFoam.Basics.Utilities import execute,rmtree,copytree
16 from PyFoam.Execution.AnalyzedRunner import AnalyzedRunner
17 from PyFoam.Execution.ConvergenceRunner import ConvergenceRunner
18 from PyFoam.Execution.BasicRunner import BasicRunner
19 from PyFoam.Execution.UtilityRunner import UtilityRunner
20 from PyFoam.Execution.ParallelExecution import LAMMachine
21 from PyFoam.RunDictionary.ParsedParameterFile import ParsedParameterFile
22 from PyFoam.LogAnalysis.BoundingLogAnalyzer import BoundingLogAnalyzer
23 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
24 from PyFoam.Basics.CSVCollection import CSVCollection
25
26 from .PyFoamApplication import PyFoamApplication
27 from PyFoam.FoamInformation import changeFoamVersion,injectVariables
28
29 from .Decomposer import Decomposer
30
31 from PyFoam.ThirdParty.six import print_,iteritems,exec_
32
44
46 solver=OptionGroup(self.parser,
47 "Solver",
48 "Controls the behaviour of the solver")
49 self.parser.add_option_group(solver)
50 result=OptionGroup(self.parser,
51 "Results",
52 "What should be done with the solver results")
53 self.parser.add_option_group(solver)
54 behave=OptionGroup(self.parser,
55 "Behaviour",
56 "What should be done and output")
57 self.parser.add_option_group(behave)
58
59 behave.add_option("--test",
60 action="store_true",
61 default=False,
62 dest="test",
63 help="Only does the preparation steps but does not execute the actual solver an the end")
64
65 result.add_option("--removeOld",
66 action="store_true",
67 default=False,
68 dest="removeOld",
69 help="Remove the directories from an old run without asking")
70
71 result.add_option("--purge",
72 action="store_true",
73 default=False,
74 dest="purge",
75 help="Remove the case directories after evaluating")
76
77 result.add_option("--no-purge",
78 action="store_true",
79 default=False,
80 dest="nopurge",
81 help="Don't remove the case directories after evaluating")
82
83 solver.add_option("--steady",
84 action="store_true",
85 default=False,
86 dest="steady",
87 help="Only runs the solver until convergence")
88
89 behave.add_option("--showDictionary",
90 action="store_true",
91 default=False,
92 dest="showDict"
93 ,help="Shows the parameter-dictionary after the running of the solver")
94
95 solver.add_option("--no-server",
96 dest="server",
97 default=True,
98 action="store_false",
99 help="Don't start the process-control-server")
100
102 fName=self.parser.getArgs()[0]
103
104 dom=parse(fName)
105 doc=dom.documentElement
106
107 if doc.tagName!='comparator':
108 error("Wrong root-element",doc.tagName,"Expected: 'comparator'")
109
110 self.data=ComparatorData(doc)
111
112 purge=False
113 if doc.hasAttribute('purge'):
114 purge=eval(doc.getAttribute('purge'))
115 if self.opts.purge:
116 purge=self.opts.purge
117 if self.opts.nopurge:
118 purge=False
119
120 steady=False
121 if doc.hasAttribute('steady'):
122 steady=eval(doc.getAttribute('steady'))
123 if self.opts.steady:
124 purge=self.opts.steady
125
126 print_(" Parameters read OK ")
127 print_()
128
129 aLog=open(self.data.id+".overview","w")
130 csv=CSVCollection(self.data.id+".csv")
131
132 rDir=self.data.id+".results"
133 rmtree(rDir)
134 mkdir(rDir)
135
136 calculated=0
137 format="%%0%dd" % len(str(len(self.data)))
138
139 for i in range(len(self.data)):
140 runID=(format % i)
141 print_(runID,end=" ",file=aLog)
142 csv["ID"]=runID
143
144 use,para=self.data[i]
145 para["template"]=self.data.template
146 para["extension"]=self.data.extension
147 para["id"]=self.data.id
148
149 if use:
150 calculated+=1
151
152 print_("Executing Variation",i+1,"of",len(self.data),end=" ")
153 if calculated!=i+1:
154 print_("(",calculated,"actually calculated)")
155 else:
156 print_()
157
158 print_("Parameters:",end=" ")
159 for k,v in iteritems(para):
160 print_("%s='%s' " % (k,v),end=" ")
161 if v.find(" ")>=0 or v.find("\t")>=0:
162 v="'"+v+"'"
163 print_(v,end=" ",file=aLog)
164 csv[k]=v
165
166 print_()
167
168 if not use:
169 print_("Skipping because not all conditions are satisfied")
170 csv.clear()
171 print_()
172 continue
173
174 cName=("%s."+format) % (self.data.id, i)
175 log=open(cName+".log","w")
176
177 para["case"]=cName
178 print_("Case-directory:",cName)
179 para["results"]=path.join(rDir,runID)
180 print_("Results directory:",para["results"])
181 mkdir(para["results"])
182
183 if path.exists(cName):
184 if self.opts.removeOld:
185 print_(" Removing old case-directory")
186 rmtree(cName)
187 else:
188 error("Case-directory",cName,"exists")
189
190 print_(" copying template")
191 out=copytree(self.data.template,cName)
192 print_("---- Copying",file=log)
193 for l in out:
194 print_(l,end=" ",file=log)
195
196 print_(" preparing")
197 ok,erg=self.data.prep.execute(para,log)
198 print_(ok,end=" ",file=aLog)
199 csv["prepare OK"]=ok
200
201 for i in range(len(erg)):
202 print_(erg[i],end=" ",file=aLog)
203 csv["Prepare %02d" % i]=erg[i]
204
205 aLog.flush()
206
207 if self.opts.test:
208 print_(" Skipping execution")
209 else:
210 print_(" running the solver")
211 sys.stdout.flush()
212
213 if steady:
214 runnerClass=ConvergenceRunner
215 else:
216 runnerClass=AnalyzedRunner
217
218 run=runnerClass(BoundingLogAnalyzer(doTimelines=True,progress=True),
219 argv=[self.data.solver,".",cName],
220 silent=True,
221 lam=Command.parallel,
222 server=self.opts.server)
223
224 run.start()
225 ok=run.runOK()
226 if ok:
227 print_(" executed OK")
228 else:
229 print_(" fatal error")
230
231 for aName in run.listAnalyzers():
232 a=run.getAnalyzer(aName)
233 if 'titles' in dir(a):
234 for tit in a.lines.getValueNames():
235 t,v=a.getTimeline(tit)
236 if len(v)>0:
237 para["result_"+aName+"_"+tit]=v[-1]
238
239 print_(run.runOK(),run.lastTime(),run.run.wallTime(),end=" ",file=aLog)
240 csv["Run OK"]=run.runOK()
241 csv["End Time"]=run.lastTime()
242 csv["Wall Time"]=run.run.wallTime()
243 csv["Wall Time (Foam)"]=run.totalClockTime()
244 csv["CPU Time"]=run.totalCpuTime()
245 csv["Wall Time First Step"]=run.firstClockTime()
246 csv["CPU Time First Step"]=run.firstCpuTime()
247
248 para["endTime"]=run.lastTime()
249 para["runlog"]=run.logFile
250
251 if self.opts.showDict:
252 print_(para)
253
254 print_(" evaluating results")
255
256 ok,erg=self.data.post.execute(para,log)
257
258 if Command.parallel!=None:
259 print_(" Stoping LAM")
260 Command.parallel.stop()
261 Command.parallel=None
262
263 if ok:
264 print_(" Evaluation OK",end=" ")
265 else:
266 print_(" Evaluation failed",end=" ")
267
268 if len(erg)>0:
269 print_(":",erg,end=" ")
270 print_()
271
272 print_(ok,end=" ",file=aLog)
273 for i in range(len(erg)):
274 print_(erg[i],end=" ",file=aLog)
275 csv["Post %02d" % i]=erg[i]
276
277 if purge:
278 print_(" removing the case-directory")
279 out=rmtree(cName)
280 print_("---- Removing",file=log)
281 for l in out:
282 print_(l,end=" ",file=log)
283
284 log.close()
285 print_()
286 print_(file=log)
287 aLog.flush()
288 csv.write()
289
290 aLog.close()
291
293 """ The object that holds the actual data"""
294
296 """
297 @param doc: the parsed XML-data from which the object is constructed
298 """
299 self.name=doc.getAttribute("name")
300 if self.name=="":
301 error("No name for 'comparator' given")
302
303 base=doc.getElementsByTagName("base")
304 if base.length!=1:
305 error("One 'base'-element needed. Found",base.length)
306 self.__parseBase(base[0])
307
308 self.vList=[]
309 for v in doc.getElementsByTagName("variation"):
310 self.vList.append(Variation(v))
311
313 """@param e: The 'base'-element"""
314
315 self.template=path.expandvars(e.getAttribute("template"))
316 if self.template=="":
317 error("No template is given")
318 if not path.exists(self.template):
319 error("Template",self.template,"does not exist")
320 self.id=path.basename(self.template)
321 if e.hasAttribute('extension'):
322 self.extension=e.getAttribute('extension')
323 self.id+="."+self.extension
324 else:
325 self.extension=""
326 self.solver=e.getAttribute("solver")
327 if self.solver=="":
328 error("No solver is given")
329 prep=e.getElementsByTagName("preparation")
330 if prep.length!=1:
331 error("One 'preparation'-element needed. Found",prep.length)
332 self.prep=PreparationChain(prep[0])
333 post=e.getElementsByTagName("evaluation")
334 if post.length!=1:
335 error("One 'evaluation'-element needed. Found",post.length)
336 self.post=EvaluationChain(post[0])
337
339 """@return: The total number of variations"""
340 if len(self.vList)==0:
341 return 0
342 else:
343 nr=1
344 for v in self.vList:
345 nr*=len(v)
346 return nr
347
349 """@param nr: Number of the variation
350 @return: dictionary with the variation"""
351 if nr>=len(self):
352 error("Index",nr,"of variation out of bounds: [0,",len(self)-1,"]")
353 result={}
354 tmp=nr
355 conditions=[]
356 for v in self.vList:
357 if (tmp % len(v))!=0:
358 conditions.append(v.condition)
359
360 k,val=v[tmp % len(v)]
361 result[k]=val
362 tmp/=len(v)
363
364 assert tmp==0
365
366 use=True
367 for c in conditions:
368 cond=replaceValues(c,result)
369 use=use and eval(cond)
370
371 return use,result
372
374 """Abstract base class for a number of commands"""
376 """@param c: XML-Subtree that represents the chain"""
377 self.commands=[]
378 for e in c.childNodes:
379 if e.nodeType!=xml.dom.Node.ELEMENT_NODE:
380 continue
381 if not e.tagName in list(self.table.keys()):
382 error("Tagname",e.tagName,"not in table of valid tags",list(self.table.keys()))
383 self.commands.append(self.table[e.tagName](e))
384
386 """Executes the chain
387 @param para:A dictionary with the parameters
388 @param log: Logfile to write to"""
389
390 result=[]
391 status=True
392 for c in self.commands:
393
394 if c.doIt(para):
395 ok,erg=c.execute(para,log)
396 else:
397 ok,erg=True,[]
398
399 status=ok and status
400 if erg!=None:
401 if type(erg)==list:
402 result+=erg
403 else:
404 result.append(erg)
405
406 return status,result
407
409 """Checks whether there is an object of a specific type"""
410
411 for o in self.commands:
412 if type(o)==typ:
413 return True
414
415 return False
416
418 """Chain of Preparation commands"""
420 self.table={"genericcommand":GenericCommand,
421 "derived":DerivedCommand,
422 "foamcommand":FoamCommand,
423 "foamutility":FoamUtilityCommand,
424 "initial":InitialCommand,
425 "dictwrite":DictWriteCommand,
426 "setdictionary":SetDictionaryCommand,
427 "decompose":DecomposeCommand,
428 "foamversion":FoamVersionCommand,
429 "changeenvironment":ChangeEnvironmentCommand,
430 "setenv":SetEnvironmentCommand,
431 "boundary":BoundaryCommand}
432 CommandChain.__init__(self,c)
433
435 """Chain of evaluation commands"""
437 self.table={"genericcommand":GenericCommand,
438 "derived":DerivedCommand,
439 "foamutility":FoamUtilityCommand,
440 "foamcommand":FoamCommand,
441 "dictionary":DictionaryCommand,
442 "reconstruct":ReconstructCommand,
443 "lastresult":LastResultCommand,
444 "changeenvironment":ChangeEnvironmentCommand,
445 "setenv":SetEnvironmentCommand,
446 "copylog":CopyLogCommand}
447 CommandChain.__init__(self,c)
448
450 result=e.getAttribute(name)
451 if result=="":
452 if default==None:
453 error("Missing attribute",name,"in element",e.tagName)
454 else:
455 return default
456 return result
457
459 """Replaces all strings enclosed by $$ with the parameters
460 @param orig: the original string
461 @param para: dictionary with the parameters"""
462
463 exp=re.compile("\$[^$]*\$")
464 tmp=orig
465
466 m=exp.search(tmp)
467 while m:
468 a,e=m.span()
469 pre=tmp[0:a]
470 post=tmp[e:]
471 mid=tmp[a+1:e-1]
472
473 if not mid in list(para.keys()):
474 error("Key",mid,"not existing in keys",list(para.keys()))
475
476 tmp=pre+para[mid]+post
477
478 m=exp.search(tmp)
479
480 return tmp
481
483
484 parallel=None
485
486 """Abstract base class of all commands"""
489
490 - def doIt(self,para):
494
496 """@param vals: Dictionary with the keywords
497 @return: A boolean whether it completed successfully and a list with results (None if no results are generated)"""
498 error("Execute not implemented for",type(self))
499
501 """Executes a shell command"""
505
507 cmd=replaceValues(self.command,para)
508 print_(" Executing ",cmd,end=" ")
509 sys.stdout.flush()
510 out=execute(cmd)
511
512 if len(out)>0:
513 print_(" -->",len(out),"lines output")
514 for l in out:
515 print_("---- Command:",cmd,file=log)
516 print_(l,end=" ",file=log)
517 else:
518 print_()
519
520 return True,None
521
523 """Derives an additional value"""
528
530 tmp=replaceValues(self.expression,para)
531 try:
532 val=eval(tmp)
533 except SyntaxError:
534 error("Syntax error in",tmp)
535 print_(" Setting",self.name,"to",val)
536 para[self.name]=str(val)
537
538 return True,None
539
541 """Returns values from the chains dictionaries"""
545
547 if self.key in para:
548 return True,para[self.key]
549 else:
550 print_("-----> ",self.key,"not in set of valid keys",list(para.keys()))
551 print_(self.key,"not in set of valid keys of dictionary",para,file=log)
552 return False,None
553
555 """Sets value in the chains dictionaries"""
560
562 para[self.key]=self.value
563 return True,None
564
566 """Changes the used OpenFOAM-version"""
570
576
578 """Sets an environment variable"""
583
591
593 """Changes Environment variables by executing a script-file"""
597
604
606 """Decomposes a case and generates the LAM"""
612
626
628 """Reconstructs a case and deleted the LAM"""
630 Command.__init__(self,c)
631 self.onlyLatest=False
632 if c.hasAttribute('onlyLatest'):
633 self.onlyLatest=eval(c.getAttribute('onlyLatest'))
634
649
651 """Executes a OpenFOAM-utility"""
656
669
671 """Executes a OpenFOAM-utility and extracts information"""
675
677 argv=[self.utility,".",para['case']]+self.options.split()
678 print_(" Executing and analyzing",string.join(argv),end=" ")
679 sys.stdout.flush()
680 run=UtilityRunner(argv,silent=True,lam=Command.parallel,logname=string.join(argv,"_"))
681 run.add("data",self.regexp)
682 run.start()
683 data=run.analyzer.getData("data")
684 result=None
685 if data!=None:
686 result=[]
687 for a in data:
688 result.append(a)
689 if result==None:
690 print_("no data",end=" ")
691 else:
692 print_(result,end=" ")
693
694 if run.runOK():
695 print_()
696 else:
697 print_("---> there was a problem")
698
699 return run.runOK(),result
700
702 """Common class for commands that operate on dictionaries"""
706
708 f=replaceValues(self.filename,para)
709 v=replaceValues(self.value,para)
710 s=replaceValues(self.subexpression,para)
711 k=replaceValues(self.key,para)
712
713 try:
714 dictFile=ParsedParameterFile(f,backup=True)
715 val=dictFile[k]
716 except KeyError:
717 self.error("Key: ",k,"not existing in File",f)
718 except IOError:
719 e = sys.exc_info()[1]
720 self.error("Problem with file",k,":",e)
721
722 try:
723 exec_("dictFile[k]"+s+"=v")
724 except Exception:
725 e = sys.exc_info()[1]
726 error("Problem with subexpression:",sys.exc_info()[0],":",e)
727
728 dictFile.writeFile()
729
730 return True,None
731
733 """Common class for commands that set values on fields"""
738
740 """Sets an initial condition"""
745
749
751 """Sets a boundary condition"""
753 FieldSetterCommand.__init__(self,c)
754 self.patch=c.getAttribute("patch")
755 self.key="boundaryField"
756 self.subexpression="['"+self.patch+"']"
757 self.element=c.getAttribute("element")
758 if self.element=="":
759 self.element="value"
760 self.subexpression+="['"+self.element+"']"
761
765
767 """Writes a value to a dictionary"""
769 SetterCommand.__init__(self,c)
770 self.dir=c.getAttribute("directory")
771 self.dict=c.getAttribute("dictionary")
772 self.filename=path.join("$case$",self.dir,self.dict)
773 self.key=c.getAttribute("key")
774 self.subexpression=c.getAttribute("subexpression")
775 self.value=c.getAttribute("value")
776
780
782 """Copies the result of the last time-step to the resultsd directory"""
785
792
794 """Copies the log file to the results"""
797
799 print_(" Copy logfile")
800 copyfile(para["runlog"],para["results"])
801 return True,None
802
804 """Represents one variation"""
805
807 """@param e: the XML-data from which it is created"""
808
809 self.name=e.getAttribute("name")
810 if self.name=="":
811 error("No name for 'variation' given")
812 self.key=e.getAttribute("key")
813 if self.key=="":
814 error("No key for 'variation'",self.name," given")
815 self.condition=e.getAttribute("condition")
816 if self.condition=="":
817 self.condition="True"
818 self.values=[]
819 for v in e.getElementsByTagName("value"):
820 self.values.append(str(v.firstChild.data))
821
823 return "Variation "+self.name+" varies "+self.key+" over "+str(self.values)
824
826 """@return: number of values"""
827 return len(self.values)
828
830 return self.key,self.values[key]
831
832
833