Package PyFoam :: Package Execution :: Module BasicRunner
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Execution.BasicRunner

  1  #  ICE Revision: $Id$ 
  2  """Run a OpenFOAM command""" 
  3   
  4  import sys 
  5  import string 
  6  import gzip 
  7  from os import path 
  8  from platform import uname 
  9  from threading import Timer 
 10  from time import time,asctime 
 11   
 12  from PyFoam.FoamInformation import oldAppConvention as oldApp 
 13  from PyFoam.ThirdParty.six import print_ 
 14  import PyFoam.Basics.FoamFileGenerator 
 15  from PyFoam.Basics.DataStructures import makePrimitiveString 
 16   
 17  if not 'curdir' in dir(path) or not 'sep' in dir(path): 
 18      print_("Warning: Inserting symbols into os.path (Python-Version<2.3)") 
 19      path.curdir='.' 
 20      path.sep   ='/' 
 21   
 22  from PyFoam.Execution.FoamThread import FoamThread 
 23  from PyFoam.Infrastructure.FoamServer import FoamServer 
 24  from PyFoam.Infrastructure.Logging import foamLogger 
 25  from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory 
 26  from PyFoam.RunDictionary.ParameterFile import ParameterFile 
 27  from PyFoam.Error import warning,error,debug 
 28  from PyFoam import configuration as config 
 29   
30 -def restoreControlDict(ctrl,runner):
31 """Timed function to avoid time-stamp-problems""" 32 warning("Restoring the controlDict") 33 ctrl.restore() 34 runner.controlDict=None
35
36 -class BasicRunner(object):
37 """Base class for the running of commands 38 39 When the command is run the output is copied to a LogFile and 40 (optionally) standard-out 41 42 The argument list assumes for the first three elements the 43 OpenFOAM-convention: 44 45 <cmd> <dir> <case> 46 47 The directory name for outputs is therefor created from <dir> and 48 <case> 49 50 Provides some handle-methods that are to be overloaded for 51 additional functionality""" 52
53 - def __init__(self, 54 argv=None, 55 silent=False, 56 logname=None, 57 compressLog=False, 58 lam=None, 59 server=False, 60 restart=False, 61 noLog=False, 62 logTail=None, 63 remark=None, 64 jobId=None, 65 parameters=None, 66 writeState=True, 67 echoCommandLine=None):
68 """@param argv: list with the tokens that are the command line 69 if not set the standard command line is used 70 @param silent: if True no output is sent to stdout 71 @param logname: name of the logfile 72 @param compressLog: Compress the logfile into a gzip 73 @param lam: Information about a parallel run 74 @param server: Whether or not to start the network-server 75 @type lam: PyFoam.Execution.ParallelExecution.LAMMachine 76 @param noLog: Don't output a log file 77 @param logTail: only the last lines of the log should be written 78 @param remark: User defined remark about the job 79 @param parameters: User defined dictionary with parameters for 80 documentation purposes 81 @param jobId: Job ID of the controlling system (Queueing system) 82 @param writeState: Write the state to some files in the case 83 @param echoCommandLine: Prefix that is printed with the command line. If unset nothing is printed 84 """ 85 86 if sys.version_info < (2,3): 87 # Python 2.2 does not have the capabilities for the Server-Thread 88 if server: 89 warning("Can not start server-process because Python-Version is too old") 90 server=False 91 92 if argv==None: 93 self.argv=sys.argv[1:] 94 else: 95 self.argv=argv 96 97 if oldApp(): 98 self.dir=path.join(self.argv[1],self.argv[2]) 99 if self.argv[2][-1]==path.sep: 100 self.argv[2]=self.argv[2][:-1] 101 else: 102 self.dir=path.curdir 103 if "-case" in self.argv: 104 self.dir=self.argv[self.argv.index("-case")+1] 105 106 if logname==None: 107 logname="PyFoam."+path.basename(argv[0]) 108 109 try: 110 sol=self.getSolutionDirectory() 111 except OSError: 112 e = sys.exc_info()[1] # compatible with 2.x and 3.x 113 error("Solution directory",self.dir,"does not exist. No use running. Problem:",e) 114 115 self.echoCommandLine=echoCommandLine 116 self.silent=silent 117 self.lam=lam 118 self.origArgv=self.argv 119 self.writeState=writeState 120 self.__lastLastSeenWrite=0 121 self.__lastNowTimeWrite=0 122 123 if self.lam!=None: 124 self.argv=lam.buildMPIrun(self.argv) 125 if config().getdebug("ParallelExecution"): 126 debug("Command line:"," ".join(self.argv)) 127 self.cmd=" ".join(self.argv) 128 foamLogger().info("Starting: "+self.cmd+" in "+path.abspath(path.curdir)) 129 self.logFile=path.join(self.dir,logname+".logfile") 130 131 self.noLog=noLog 132 self.logTail=logTail 133 if self.logTail: 134 if self.noLog: 135 warning("Log tail",self.logTail,"and no-log specified. Using logTail") 136 self.noLog=True 137 self.lastLines=[] 138 139 self.compressLog=compressLog 140 if self.compressLog: 141 self.logFile+=".gz" 142 143 self.fatalError=False 144 self.fatalFPE=False 145 self.fatalStackdump=False 146 147 self.warnings=0 148 self.started=False 149 150 self.isRestarted=False 151 if restart: 152 self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True) 153 self.controlDict.replaceParameter("startFrom","latestTime") 154 self.isRestarted=True 155 else: 156 self.controlDict=None 157 158 self.run=FoamThread(self.cmd,self) 159 160 self.server=None 161 if server: 162 self.server=FoamServer(run=self.run,master=self) 163 self.server.setDaemon(True) 164 self.server.start() 165 try: 166 IP,PID,Port=self.server.info() 167 f=open(path.join(self.dir,"PyFoamServer.info"),"w") 168 print_(IP,PID,Port,file=f) 169 f.close() 170 except AttributeError: 171 warning("There seems to be a problem with starting the server:",self.server,"with attributes",dir(self.server)) 172 self.server=None 173 174 self.createTime=None 175 self.nowTime=None 176 self.startTimestamp=time() 177 178 self.stopMe=False 179 self.writeRequested=False 180 181 self.endTriggers=[] 182 183 self.lastLogLineSeen=None 184 self.lastTimeStepSeen=None 185 186 self.remark=remark 187 self.jobId=jobId 188 189 self.data={"lines":0} # self.data={"lines":0L} 190 self.data["logfile"]=self.logFile 191 self.data["casefullname"]=path.abspath(self.dir) 192 self.data["casename"]=path.basename(path.abspath(self.dir)) 193 self.data["solver"]=path.basename(self.argv[0]) 194 self.data["solverFull"]=self.argv[0] 195 self.data["commandLine"]=self.cmd 196 self.data["hostname"]=uname()[1] 197 if remark: 198 self.data["remark"]=remark 199 else: 200 self.data["remark"]="No remark given" 201 if jobId: 202 self.data["jobId"]=jobId 203 parameterFile=sol.getParametersFromFile() 204 if len(parameterFile): 205 self.data["parameters"]={} 206 for k,v in parameterFile.items(): 207 self.data["parameters"][k]=makePrimitiveString(v) 208 if parameters: 209 if "parameters" not in self.data: 210 self.data["parameters"]={} 211 self.data["parameters"].update(parameters) 212 self.data["starttime"]=asctime()
213
214 - def appendTailLine(self,line):
215 """Append lines to the tail of the log""" 216 if len(self.lastLines)>10*self.logTail: 217 # truncate the lines, but not too often 218 self.lastLines=self.lastLines[-self.logTail:] 219 self.writeTailLog() 220 221 self.lastLines.append(line+"\n")
222
223 - def writeTailLog(self):
224 """Write the last lines to the log""" 225 fh=open(self.logFile,"w") 226 if len(self.lastLines)<=self.logTail: 227 fh.writelines(self.lastLines) 228 else: 229 fh.writelines(self.lastLines[-self.logTail:]) 230 fh.close()
231
232 - def start(self):
233 """starts the command and stays with it till the end""" 234 235 self.started=True 236 if not self.noLog: 237 if self.compressLog: 238 fh=gzip.open(self.logFile,"w") 239 else: 240 fh=open(self.logFile,"w") 241 242 self.startHandle() 243 244 self.writeStartTime() 245 self.writeTheState("Running") 246 247 check=BasicRunnerCheck() 248 249 if self.echoCommandLine: 250 print_(self.echoCommandLine+" "+" ".join(self.argv)) 251 252 self.run.start() 253 interrupted=False 254 255 totalWarningLines=0 256 addLinesToWarning=0 257 collectWarnings=True 258 259 while self.run.check(): 260 try: 261 self.run.read() 262 if not self.run.check(): 263 break 264 265 line=self.run.getLine() 266 267 if "errorText" in self.data: 268 self.data["errorText"]+=line+"\n" 269 270 if addLinesToWarning>0: 271 self.data["warningText"]+=line+"\n" 272 addLinesToWarning-=1 273 totalWarningLines+=1 274 if totalWarningLines>500: 275 collectWarnings=False 276 addLinesToWarning=0 277 self.data["warningText"]+="No more warnings added because limit of 500 lines exceeded" 278 self.data["lines"]+=1 279 self.lastLogLineSeen=time() 280 self.writeLastSeen() 281 282 tmp=check.getTime(line) 283 if check.controlDictRead(line): 284 if self.writeRequested: 285 duration=config().getfloat("Execution","controlDictRestoreWait",default=30.) 286 warning("Preparing to reset controlDict to old glory in",duration,"seconds") 287 Timer(duration, 288 restoreControlDict, 289 args=[self.controlDict,self]).start() 290 self.writeRequested=False 291 292 if tmp!=None: 293 self.data["time"]=tmp 294 self.nowTime=tmp 295 self.writeTheState("Running",always=False) 296 self.writeNowTime() 297 self.lastTimeStepSeen=time() 298 if self.createTime==None: 299 # necessary because interFoam reports no creation time 300 self.createTime=tmp 301 try: 302 self.data["stepNr"]+=1 303 except KeyError: 304 self.data["stepNr"]=1 # =1L 305 306 self.data["lasttimesteptime"]=asctime() 307 308 tmp=check.getCreateTime(line) 309 if tmp!=None: 310 self.createTime=tmp 311 312 if not self.silent: 313 try: 314 print_(line) 315 except IOError: 316 e = sys.exc_info()[1] # compatible with 2.x and 3.x 317 if e.errno!=32: 318 raise e 319 else: 320 # Pipe was broken 321 self.run.interrupt() 322 323 if line.find("FOAM FATAL ERROR")>=0 or line.find("FOAM FATAL IO ERROR")>=0: 324 self.fatalError=True 325 self.data["errorText"]="PyFoam found a Fatal Error " 326 if "time" in self.data: 327 self.data["errorText"]+="at time "+str(self.data["time"])+"\n" 328 else: 329 self.data["errorText"]+="before time started\n" 330 self.data["errorText"]+="\n"+line+"\n" 331 332 if line.find("Foam::sigFpe::sigFpeHandler")>=0: 333 self.fatalFPE=True 334 if line.find("Foam::error::printStack")>=0: 335 self.fatalStackdump=True 336 337 if self.fatalError and line!="": 338 foamLogger().error(line) 339 340 if line.find("FOAM Warning")>=0: 341 self.warnings+=1 342 try: 343 self.data["warnings"]+=1 344 except KeyError: 345 self.data["warnings"]=1 346 if collectWarnings: 347 addLinesToWarning=20 348 if not "warningText" in self.data: 349 self.data["warningText"]="" 350 else: 351 self.data["warningText"]+=("-"*40)+"\n" 352 self.data["warningText"]+="Warning found by PyFoam on line " 353 self.data["warningText"]+=str(self.data["lines"])+" " 354 if "time" in self.data: 355 self.data["warningText"]+="at time "+str(self.data["time"])+"\n" 356 else: 357 self.data["warningText"]+="before time started\n" 358 self.data["warningText"]+="\n"+line+"\n" 359 360 if self.server!=None: 361 self.server._insertLine(line) 362 363 self.lineHandle(line) 364 365 if not self.noLog: 366 fh.write(line+"\n") 367 fh.flush() 368 elif self.logTail: 369 self.appendTailLine(line) 370 371 except KeyboardInterrupt: 372 e = sys.exc_info()[1] # compatible with 2.x and 3.x 373 foamLogger().warning("Keyboard Interrupt") 374 self.run.interrupt() 375 self.writeTheState("Interrupted") 376 interrupted=True 377 378 self.data["interrupted"]=interrupted 379 self.data["OK"]=self.runOK() 380 self.data["cpuTime"]=self.run.cpuTime() 381 self.data["cpuUserTime"]=self.run.cpuUserTime() 382 self.data["cpuSystemTime"]=self.run.cpuSystemTime() 383 self.data["wallTime"]=self.run.wallTime() 384 self.data["usedMemory"]=self.run.usedMemory() 385 self.data["endtime"]=asctime() 386 387 self.data["fatalError"]=self.fatalError 388 self.data["fatalFPE"]=self.fatalFPE 389 self.data["fatalStackdump"]=self.fatalStackdump 390 391 self.writeNowTime(force=True) 392 393 self.stopHandle() 394 395 if not interrupted: 396 self.writeTheState("Finished") 397 398 for t in self.endTriggers: 399 t() 400 401 if not self.noLog: 402 fh.close() 403 elif self.logTail: 404 self.writeTailLog() 405 406 if self.server!=None: 407 self.server.deregister() 408 self.server.kill() 409 410 foamLogger().info("Finished") 411 412 return self.data
413
414 - def writeToStateFile(self,fName,message):
415 """Write a message to a state file""" 416 if self.writeState: 417 open(path.join(self.dir,"PyFoamState."+fName),"w").write(message+"\n")
418
419 - def writeStartTime(self):
420 """Write the real time the run was started at""" 421 self.writeToStateFile("StartedAt",asctime())
422
423 - def writeTheState(self,state,always=True):
424 """Write the current state the run is in""" 425 if always or (time()-self.__lastLastSeenWrite)>9: 426 self.writeToStateFile("TheState",state)
427
428 - def writeLastSeen(self):
429 if (time()-self.__lastLastSeenWrite)>10: 430 self.writeToStateFile("LastOutputSeen",asctime()) 431 self.__lastLastSeenWrite=time()
432
433 - def writeNowTime(self,force=False):
434 if (time()-self.__lastNowTimeWrite)>10 or force: 435 self.writeToStateFile("CurrentTime",str(self.nowTime)) 436 self.__lastNowTimeWrite=time()
437
438 - def runOK(self):
439 """checks whether the run was successful""" 440 if self.started: 441 return not self.fatalError and not self.fatalFPE and not self.fatalStackdump # and self.run.getReturnCode()==0 442 else: 443 return False
444
445 - def startHandle(self):
446 """to be called before the program is started""" 447 pass
448
449 - def stopGracefully(self):
450 """Tells the runner to stop at the next convenient time""" 451 if not self.stopMe: 452 self.stopMe=True 453 if not self.isRestarted: 454 if self.controlDict: 455 warning("The controlDict has already been modified. Restoring will be problementic") 456 self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True) 457 self.controlDict.replaceParameter("stopAt","writeNow") 458 warning("Stopping run at next write")
459
460 - def writeResults(self):
461 """Writes the next possible time-step""" 462 # warning("writeResult is not yet implemented") 463 if not self.writeRequested: 464 if not self.isRestarted: 465 if self.controlDict: 466 warning("The controlDict has already been modified. Restoring will be problementic") 467 self.controlDict=ParameterFile(path.join(self.dir,"system","controlDict"),backup=True) 468 self.controlDict.replaceParameter("writeControl","timeStep") 469 self.controlDict.replaceParameter("writeInterval","1") 470 self.writeRequested=True
471
472 - def stopHandle(self):
473 """called after the program has stopped""" 474 if self.stopMe or self.isRestarted: 475 self.controlDict.restore()
476
477 - def lineHandle(self,line):
478 """called every time a new line is read""" 479 pass
480
481 - def logName(self):
482 """Get the name of the logfiles""" 483 return self.logFile
484
485 - def getSolutionDirectory(self,archive=None):
486 """@return: The directory of the case 487 @rtype: PyFoam.RunDictionary.SolutionDirectory 488 @param archive: Name of the directory for archiving results""" 489 490 return SolutionDirectory(self.dir,archive=archive,parallel=True)
491
492 - def addEndTrigger(self,f):
493 """@param f: A function that is to be executed at the end of the simulation""" 494 self.endTriggers.append(f)
495 496 import re 497
498 -class BasicRunnerCheck(object):
499 """A small class that does primitve checking for BasicRunner 500 Duplicates other efforts, but ....""" 501 502 floatRegExp="[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?" 503
504 - def __init__(self):
505 # self.timeExpr=re.compile("^Time = (%f%)$".replace("%f%",self.floatRegExp)) 506 self.timeExpr=config().getRegexp("SolverOutput","timeregexp") 507 self.createExpr=re.compile("^Create mesh for time = (%f%)$".replace("%f%",self.floatRegExp))
508
509 - def getTime(self,line):
510 """Does this line contain time information?""" 511 m=self.timeExpr.match(line) 512 if m: 513 try: 514 return float(m.group(2)) 515 except ValueError: 516 warning("Problem while converting",m.group(2),"to float") 517 return None 518 else: 519 return None
520
521 - def getCreateTime(self,line):
522 """Does this line contain mesh time information?""" 523 m=self.createExpr.match(line) 524 if m: 525 return float(m.group(1)) 526 else: 527 return None
528
529 - def controlDictRead(self,line):
530 """Was the controlDict reread?""" 531 phrases=["Reading object controlDict from file", 532 "Re-reading object controlDict from file"] 533 534 for p in phrases: 535 if line.find(p)>=0: 536 return True 537 538 return False
539 540 # Should work with Python3 and Python2 541