1
2 """
3 Application class that implements pyFoamComparator
4 """
5
6 import sys
7 import re
8 import string
9 from xml.dom.minidom import parse
10 import xml.dom
11 from os import path,environ,mkdir
12 from optparse import OptionGroup
13
14 from PyFoam.Error import error
15 from PyFoam.Basics.Utilities import execute,rmtree,copytree
16 from PyFoam.Execution.AnalyzedRunner import AnalyzedRunner
17 from PyFoam.Execution.ConvergenceRunner import ConvergenceRunner
18 from PyFoam.Execution.BasicRunner import BasicRunner
19 from PyFoam.Execution.UtilityRunner import UtilityRunner
20 from PyFoam.Execution.ParallelExecution import LAMMachine
21 from PyFoam.RunDictionary.ParsedParameterFile import ParsedParameterFile
22 from PyFoam.LogAnalysis.BoundingLogAnalyzer import BoundingLogAnalyzer
23 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
24 from PyFoam.Basics.CSVCollection import CSVCollection
25
26 from .PyFoamApplication import PyFoamApplication
27 from PyFoam.FoamInformation import changeFoamVersion,injectVariables
28
29 from .Decomposer import Decomposer
30
31 from PyFoam.ThirdParty.six import print_,iteritems,exec_
32
34 - def __init__(self,
35 args=None,
36 **kwargs):
37 description="""\
38 Reads an XML-file that specifies a base case and a parameter-variation
39 and executes all the variations of that case
40 """
41
42 PyFoamApplication.__init__(self,
43 args=args,
44 description=description,
45 usage="%prog [options] <xmlfile>",
46 nr=1,
47 interspersed=True,
48 **kwargs)
49
51 solver=OptionGroup(self.parser,
52 "Solver",
53 "Controls the behaviour of the solver")
54 self.parser.add_option_group(solver)
55 result=OptionGroup(self.parser,
56 "Results",
57 "What should be done with the solver results")
58 self.parser.add_option_group(solver)
59 behave=OptionGroup(self.parser,
60 "Behaviour",
61 "What should be done and output")
62 self.parser.add_option_group(behave)
63
64 behave.add_option("--test",
65 action="store_true",
66 default=False,
67 dest="test",
68 help="Only does the preparation steps but does not execute the actual solver an the end")
69
70 result.add_option("--removeOld",
71 action="store_true",
72 default=False,
73 dest="removeOld",
74 help="Remove the directories from an old run without asking")
75
76 result.add_option("--purge",
77 action="store_true",
78 default=False,
79 dest="purge",
80 help="Remove the case directories after evaluating")
81
82 result.add_option("--no-purge",
83 action="store_true",
84 default=False,
85 dest="nopurge",
86 help="Don't remove the case directories after evaluating")
87
88 solver.add_option("--steady",
89 action="store_true",
90 default=False,
91 dest="steady",
92 help="Only runs the solver until convergence")
93
94 behave.add_option("--showDictionary",
95 action="store_true",
96 default=False,
97 dest="showDict"
98 ,help="Shows the parameter-dictionary after the running of the solver")
99
100 solver.add_option("--no-server",
101 dest="server",
102 default=True,
103 action="store_false",
104 help="Don't start the process-control-server")
105
107 fName=self.parser.getArgs()[0]
108
109 dom=parse(fName)
110 doc=dom.documentElement
111
112 if doc.tagName!='comparator':
113 error("Wrong root-element",doc.tagName,"Expected: 'comparator'")
114
115 self.data=ComparatorData(doc)
116
117 purge=False
118 if doc.hasAttribute('purge'):
119 purge=eval(doc.getAttribute('purge'))
120 if self.opts.purge:
121 purge=self.opts.purge
122 if self.opts.nopurge:
123 purge=False
124
125 steady=False
126 if doc.hasAttribute('steady'):
127 steady=eval(doc.getAttribute('steady'))
128 if self.opts.steady:
129 purge=self.opts.steady
130
131 print_(" Parameters read OK ")
132 print_()
133
134 aLog=open(self.data.id+".overview","w")
135 csv=CSVCollection(self.data.id+".csv")
136
137 rDir=self.data.id+".results"
138 rmtree(rDir)
139 mkdir(rDir)
140
141 calculated=0
142 format="%%0%dd" % len(str(len(self.data)))
143
144 for i in range(len(self.data)):
145 runID=(format % i)
146 print_(runID,end=" ",file=aLog)
147 csv["ID"]=runID
148
149 use,para=self.data[i]
150 para["template"]=self.data.template
151 para["extension"]=self.data.extension
152 para["id"]=self.data.id
153
154 if use:
155 calculated+=1
156
157 print_("Executing Variation",i+1,"of",len(self.data),end=" ")
158 if calculated!=i+1:
159 print_("(",calculated,"actually calculated)")
160 else:
161 print_()
162
163 print_("Parameters:",end=" ")
164 for k,v in iteritems(para):
165 print_("%s='%s' " % (k,v),end=" ")
166 if v.find(" ")>=0 or v.find("\t")>=0:
167 v="'"+v+"'"
168 print_(v,end=" ",file=aLog)
169 csv[k]=v
170
171 print_()
172
173 if not use:
174 print_("Skipping because not all conditions are satisfied")
175 csv.clear()
176 print_()
177 continue
178
179 cName=("%s."+format) % (self.data.id, i)
180 log=open(cName+".log","w")
181
182 para["case"]=cName
183 print_("Case-directory:",cName)
184 para["results"]=path.join(rDir,runID)
185 print_("Results directory:",para["results"])
186 mkdir(para["results"])
187
188 if path.exists(cName):
189 if self.opts.removeOld:
190 print_(" Removing old case-directory")
191 rmtree(cName)
192 else:
193 error("Case-directory",cName,"exists")
194
195 print_(" copying template")
196 out=copytree(self.data.template,cName)
197 print_("---- Copying",file=log)
198 for l in out:
199 print_(l,end=" ",file=log)
200
201 print_(" preparing")
202 ok,erg=self.data.prep.execute(para,log)
203 print_(ok,end=" ",file=aLog)
204 csv["prepare OK"]=ok
205
206 for i in range(len(erg)):
207 print_(erg[i],end=" ",file=aLog)
208 csv["Prepare %02d" % i]=erg[i]
209
210 aLog.flush()
211
212 if self.opts.test:
213 print_(" Skipping execution")
214 else:
215 print_(" running the solver")
216 sys.stdout.flush()
217
218 if steady:
219 runnerClass=ConvergenceRunner
220 else:
221 runnerClass=AnalyzedRunner
222
223 run=runnerClass(BoundingLogAnalyzer(doTimelines=True,progress=True),
224 argv=[self.data.solver,".",cName],
225 silent=True,
226 lam=Command.parallel,
227 server=self.opts.server)
228
229 run.start()
230 ok=run.runOK()
231 if ok:
232 print_(" executed OK")
233 else:
234 print_(" fatal error")
235
236 for aName in run.listAnalyzers():
237 a=run.getAnalyzer(aName)
238 if 'titles' in dir(a):
239 for tit in a.lines.getValueNames():
240 t,v=a.getTimeline(tit)
241 if len(v)>0:
242 para["result_"+aName+"_"+tit]=v[-1]
243
244 print_(run.runOK(),run.lastTime(),run.run.wallTime(),end=" ",file=aLog)
245 csv["Run OK"]=run.runOK()
246 csv["End Time"]=run.lastTime()
247 csv["Wall Time"]=run.run.wallTime()
248 csv["Wall Time (Foam)"]=run.totalClockTime()
249 csv["CPU Time"]=run.totalCpuTime()
250 csv["Wall Time First Step"]=run.firstClockTime()
251 csv["CPU Time First Step"]=run.firstCpuTime()
252
253 para["endTime"]=run.lastTime()
254 para["runlog"]=run.logFile
255
256 if self.opts.showDict:
257 print_(para)
258
259 print_(" evaluating results")
260
261 ok,erg=self.data.post.execute(para,log)
262
263 if Command.parallel!=None:
264 print_(" Stoping LAM")
265 Command.parallel.stop()
266 Command.parallel=None
267
268 if ok:
269 print_(" Evaluation OK",end=" ")
270 else:
271 print_(" Evaluation failed",end=" ")
272
273 if len(erg)>0:
274 print_(":",erg,end=" ")
275 print_()
276
277 print_(ok,end=" ",file=aLog)
278 for i in range(len(erg)):
279 print_(erg[i],end=" ",file=aLog)
280 csv["Post %02d" % i]=erg[i]
281
282 if purge:
283 print_(" removing the case-directory")
284 out=rmtree(cName)
285 print_("---- Removing",file=log)
286 for l in out:
287 print_(l,end=" ",file=log)
288
289 log.close()
290 print_()
291 print_(file=log)
292 aLog.flush()
293 csv.write()
294
295 aLog.close()
296
298 """ The object that holds the actual data"""
299
301 """
302 @param doc: the parsed XML-data from which the object is constructed
303 """
304 self.name=doc.getAttribute("name")
305 if self.name=="":
306 error("No name for 'comparator' given")
307
308 base=doc.getElementsByTagName("base")
309 if base.length!=1:
310 error("One 'base'-element needed. Found",base.length)
311 self.__parseBase(base[0])
312
313 self.vList=[]
314 for v in doc.getElementsByTagName("variation"):
315 self.vList.append(Variation(v))
316
318 """@param e: The 'base'-element"""
319
320 self.template=path.expandvars(e.getAttribute("template"))
321 if self.template=="":
322 error("No template is given")
323 if not path.exists(self.template):
324 error("Template",self.template,"does not exist")
325 self.id=path.basename(self.template)
326 if e.hasAttribute('extension'):
327 self.extension=e.getAttribute('extension')
328 self.id+="."+self.extension
329 else:
330 self.extension=""
331 self.solver=e.getAttribute("solver")
332 if self.solver=="":
333 error("No solver is given")
334 prep=e.getElementsByTagName("preparation")
335 if prep.length!=1:
336 error("One 'preparation'-element needed. Found",prep.length)
337 self.prep=PreparationChain(prep[0])
338 post=e.getElementsByTagName("evaluation")
339 if post.length!=1:
340 error("One 'evaluation'-element needed. Found",post.length)
341 self.post=EvaluationChain(post[0])
342
344 """@return: The total number of variations"""
345 if len(self.vList)==0:
346 return 0
347 else:
348 nr=1
349 for v in self.vList:
350 nr*=len(v)
351 return nr
352
354 """@param nr: Number of the variation
355 @return: dictionary with the variation"""
356 if nr>=len(self):
357 error("Index",nr,"of variation out of bounds: [0,",len(self)-1,"]")
358 result={}
359 tmp=nr
360 conditions=[]
361 for v in self.vList:
362 if (tmp % len(v))!=0:
363 conditions.append(v.condition)
364
365 k,val=v[tmp % len(v)]
366 result[k]=val
367 tmp/=len(v)
368
369 assert tmp==0
370
371 use=True
372 for c in conditions:
373 cond=replaceValues(c,result)
374 use=use and eval(cond)
375
376 return use,result
377
379 """Abstract base class for a number of commands"""
381 """@param c: XML-Subtree that represents the chain"""
382 self.commands=[]
383 for e in c.childNodes:
384 if e.nodeType!=xml.dom.Node.ELEMENT_NODE:
385 continue
386 if not e.tagName in list(self.table.keys()):
387 error("Tagname",e.tagName,"not in table of valid tags",list(self.table.keys()))
388 self.commands.append(self.table[e.tagName](e))
389
391 """Executes the chain
392 @param para:A dictionary with the parameters
393 @param log: Logfile to write to"""
394
395 result=[]
396 status=True
397 for c in self.commands:
398
399 if c.doIt(para):
400 ok,erg=c.execute(para,log)
401 else:
402 ok,erg=True,[]
403
404 status=ok and status
405 if erg!=None:
406 if type(erg)==list:
407 result+=erg
408 else:
409 result.append(erg)
410
411 return status,result
412
414 """Checks whether there is an object of a specific type"""
415
416 for o in self.commands:
417 if type(o)==typ:
418 return True
419
420 return False
421
423 """Chain of Preparation commands"""
425 self.table={"genericcommand":GenericCommand,
426 "derived":DerivedCommand,
427 "foamcommand":FoamCommand,
428 "foamutility":FoamUtilityCommand,
429 "initial":InitialCommand,
430 "dictwrite":DictWriteCommand,
431 "setdictionary":SetDictionaryCommand,
432 "decompose":DecomposeCommand,
433 "foamversion":FoamVersionCommand,
434 "changeenvironment":ChangeEnvironmentCommand,
435 "setenv":SetEnvironmentCommand,
436 "boundary":BoundaryCommand}
437 CommandChain.__init__(self,c)
438
440 """Chain of evaluation commands"""
442 self.table={"genericcommand":GenericCommand,
443 "derived":DerivedCommand,
444 "foamutility":FoamUtilityCommand,
445 "foamcommand":FoamCommand,
446 "dictionary":DictionaryCommand,
447 "reconstruct":ReconstructCommand,
448 "lastresult":LastResultCommand,
449 "changeenvironment":ChangeEnvironmentCommand,
450 "setenv":SetEnvironmentCommand,
451 "copylog":CopyLogCommand}
452 CommandChain.__init__(self,c)
453
455 result=e.getAttribute(name)
456 if result=="":
457 if default==None:
458 error("Missing attribute",name,"in element",e.tagName)
459 else:
460 return default
461 return result
462
464 """Replaces all strings enclosed by $$ with the parameters
465 @param orig: the original string
466 @param para: dictionary with the parameters"""
467
468 exp=re.compile("\$[^$]*\$")
469 tmp=orig
470
471 m=exp.search(tmp)
472 while m:
473 a,e=m.span()
474 pre=tmp[0:a]
475 post=tmp[e:]
476 mid=tmp[a+1:e-1]
477
478 if not mid in list(para.keys()):
479 error("Key",mid,"not existing in keys",list(para.keys()))
480
481 tmp=pre+para[mid]+post
482
483 m=exp.search(tmp)
484
485 return tmp
486
488
489 parallel=None
490
491 """Abstract base class of all commands"""
494
495 - def doIt(self,para):
499
501 """@param vals: Dictionary with the keywords
502 @return: A boolean whether it completed successfully and a list with results (None if no results are generated)"""
503 error("Execute not implemented for",type(self))
504
506 """Executes a shell command"""
510
512 cmd=replaceValues(self.command,para)
513 print_(" Executing ",cmd,end=" ")
514 sys.stdout.flush()
515 out=execute(cmd)
516
517 if len(out)>0:
518 print_(" -->",len(out),"lines output")
519 for l in out:
520 print_("---- Command:",cmd,file=log)
521 print_(l,end=" ",file=log)
522 else:
523 print_()
524
525 return True,None
526
528 """Derives an additional value"""
533
535 tmp=replaceValues(self.expression,para)
536 try:
537 val=eval(tmp)
538 except SyntaxError:
539 error("Syntax error in",tmp)
540 print_(" Setting",self.name,"to",val)
541 para[self.name]=str(val)
542
543 return True,None
544
546 """Returns values from the chains dictionaries"""
550
552 if self.key in para:
553 return True,para[self.key]
554 else:
555 print_("-----> ",self.key,"not in set of valid keys",list(para.keys()))
556 print_(self.key,"not in set of valid keys of dictionary",para,file=log)
557 return False,None
558
560 """Sets value in the chains dictionaries"""
565
567 para[self.key]=self.value
568 return True,None
569
571 """Changes the used OpenFOAM-version"""
575
581
583 """Sets an environment variable"""
588
596
598 """Changes Environment variables by executing a script-file"""
602
609
611 """Decomposes a case and generates the LAM"""
617
631
633 """Reconstructs a case and deleted the LAM"""
635 Command.__init__(self,c)
636 self.onlyLatest=False
637 if c.hasAttribute('onlyLatest'):
638 self.onlyLatest=eval(c.getAttribute('onlyLatest'))
639
654
656 """Executes a OpenFOAM-utility"""
661
674
676 """Executes a OpenFOAM-utility and extracts information"""
680
682 argv=[self.utility,".",para['case']]+self.options.split()
683 print_(" Executing and analyzing"," ".join(argv),end=" ")
684 sys.stdout.flush()
685 run=UtilityRunner(argv,silent=True,lam=Command.parallel,logname="_".join(argv))
686 run.add("data",self.regexp)
687 run.start()
688 data=run.analyzer.getData("data")
689 result=None
690 if data!=None:
691 result=[]
692 for a in data:
693 result.append(a)
694 if result==None:
695 print_("no data",end=" ")
696 else:
697 print_(result,end=" ")
698
699 if run.runOK():
700 print_()
701 else:
702 print_("---> there was a problem")
703
704 return run.runOK(),result
705
707 """Common class for commands that operate on dictionaries"""
711
713 f=replaceValues(self.filename,para)
714 v=replaceValues(self.value,para)
715 s=replaceValues(self.subexpression,para)
716 k=replaceValues(self.key,para)
717
718 try:
719 dictFile=ParsedParameterFile(f,backup=True)
720 val=dictFile[k]
721 except KeyError:
722 self.error("Key: ",k,"not existing in File",f)
723 except IOError:
724 e = sys.exc_info()[1]
725 self.error("Problem with file",k,":",e)
726
727 try:
728 exec_("dictFile[k]"+s+"=v")
729 except Exception:
730 e = sys.exc_info()[1]
731 error("Problem with subexpression:",sys.exc_info()[0],":",e)
732
733 dictFile.writeFile()
734
735 return True,None
736
738 """Common class for commands that set values on fields"""
743
745 """Sets an initial condition"""
750
754
756 """Sets a boundary condition"""
758 FieldSetterCommand.__init__(self,c)
759 self.patch=c.getAttribute("patch")
760 self.key="boundaryField"
761 self.subexpression="['"+self.patch+"']"
762 self.element=c.getAttribute("element")
763 if self.element=="":
764 self.element="value"
765 self.subexpression+="['"+self.element+"']"
766
770
772 """Writes a value to a dictionary"""
774 SetterCommand.__init__(self,c)
775 self.dir=c.getAttribute("directory")
776 self.dict=c.getAttribute("dictionary")
777 self.filename=path.join("$case$",self.dir,self.dict)
778 self.key=c.getAttribute("key")
779 self.subexpression=c.getAttribute("subexpression")
780 self.value=c.getAttribute("value")
781
785
787 """Copies the result of the last time-step to the resultsd directory"""
790
797
799 """Copies the log file to the results"""
802
804 print_(" Copy logfile")
805 copyfile(para["runlog"],para["results"])
806 return True,None
807
809 """Represents one variation"""
810
812 """@param e: the XML-data from which it is created"""
813
814 self.name=e.getAttribute("name")
815 if self.name=="":
816 error("No name for 'variation' given")
817 self.key=e.getAttribute("key")
818 if self.key=="":
819 error("No key for 'variation'",self.name," given")
820 self.condition=e.getAttribute("condition")
821 if self.condition=="":
822 self.condition="True"
823 self.values=[]
824 for v in e.getElementsByTagName("value"):
825 self.values.append(str(v.firstChild.data))
826
828 return "Variation "+self.name+" varies "+self.key+" over "+str(self.values)
829
831 """@return: number of values"""
832 return len(self.values)
833
835 return self.key,self.values[key]
836
837
838