1 """
2 Application-class that implements pyFoamListCases.py
3 """
4 from optparse import OptionGroup
5 from os import path,listdir,stat
6 from glob import glob
7 from PyFoam.ThirdParty.six.moves import cPickle as pickle
8 from PyFoam.ThirdParty.six import string_types
9 import time,datetime
10 from stat import ST_MTIME
11 import subprocess
12 import re
13 import os
14
15 from .PyFoamApplication import PyFoamApplication
16
17 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
18 from PyFoam.RunDictionary.ParsedParameterFile import ParsedParameterFile,PyFoamParserError
19
20 from PyFoam import configuration
21
22 from PyFoam.ThirdParty.six import print_,iteritems,PY3
23
24 from PyFoam.Basics.Utilities import humanReadableSize
25
26 if PY3:
27 long=int
28
30 - def __init__(self,
31 args=None,
32 **kwargs):
33 description="""\
34 List the valid OpenFOAM-cases in a number of directories along with
35 some basic information (number of timesteps, last timestep,
36 etc). Currently doesn't honor the parallel data
37 """
38 PyFoamApplication.__init__(self,
39 args=args,
40 description=description,
41 usage="%prog [<directories>]",
42 interspersed=True,
43 changeVersion=False,
44 nr=0,
45 exactNr=False,
46 **kwargs)
47
48 sortChoices=["name","first","last","mtime","nrSteps","procs","diskusage","pFirst","pLast","nrParallel","nowTime","state","lastOutput","startedAt"]
49
51 what=OptionGroup(self.parser,
52 "What",
53 "Define what should be shown")
54 self.parser.add_option_group(what)
55
56 what.add_option("--dump",
57 action="store_true",
58 dest="dump",
59 default=False,
60 help="Dump the information as Python-dictionaries")
61
62 what.add_option("--disk-usage",
63 action="store_true",
64 dest="diskusage",
65 default=False,
66 help="Show the disk-usage of the case (in MB) - may take a long time")
67
68 what.add_option("--parallel-info",
69 action="store_true",
70 dest="parallel",
71 default=False,
72 help="Print information about parallel runs (if present): number of processors and processor first and last time. The mtime will be that of the processor-directories")
73
74 what.add_option("--no-state",
75 action="store_false",
76 dest="state",
77 default=True,
78 help="Don't read state-files")
79
80 what.add_option("--advanced-state",
81 action="store_true",
82 dest="advancedState",
83 default=False,
84 help="Additional state information (run started, last output seen)")
85
86 what.add_option("--estimate-end-time",
87 action="store_true",
88 dest="estimateEndTime",
89 default=False,
90 help="Print an estimated end time (calculated from the start time of the run, the current time and the current simulation time)")
91
92 what.add_option("--start-end-time",
93 action="store_true",
94 dest="startEndTime",
95 default=False,
96 help="Start and end time from the controlDict")
97
98 what.add_option("--custom-data",
99 action="append",
100 dest="customData",
101 default=[],
102 help="Specification of additional data that is read from the pickled data-sets. The format is 'name=spec1::spec2::...' where 'name' is the name under which the data ist shown and 'specN' are the dictionary keys under which the data is accessed. If only 'spec1::spec2::..' is given then a name of the form 'CustomN' will be used. Can be specified more than once")
103
104 what.add_option("--solver-name-for-custom-data",
105 action="store",
106 dest="solverNameForCustom",
107 default=None,
108 help="This is used if '--custom-data' is specified as the data will be searched in 'PyFoamRunner.<solver name>.analyzed'. If unset then the utility will try to automatically determine the name of the solver which might be wrong")
109
110 how=OptionGroup(self.parser,
111 "How",
112 "How the things should be shown")
113 self.parser.add_option_group(how)
114
115 how.add_option("--sort-by",
116 type="choice",
117 action="store",
118 dest="sort",
119 default=configuration().get("CommandOptionDefaults","sortListCases",default="name"),
120 choices=self.sortChoices,
121 help="Sort the cases by a specific key (Keys: "+", ".join(self.sortChoices)+") Default: %default")
122 how.add_option("--reverse-sort",
123 action="store_true",
124 dest="reverse",
125 default=False,
126 help="Sort in reverse order")
127 how.add_option("--relative-times",
128 action="store_true",
129 dest="relativeTime",
130 default=False,
131 help="Show the timestamps relative to the current time")
132
133 behave=OptionGroup(self.parser,
134 "Behaviour",
135 "Additional output etc")
136 self.parser.add_option_group(behave)
137
138 behave.add_option("--progress",
139 action="store_true",
140 dest="progress",
141 default=False,
142 help="Print the directories while they are being processed")
143
151
153 dirs=self.parser.getArgs()
154
155 if len(dirs)==0:
156 dirs=[path.curdir]
157
158 cData=[]
159 totalDiskusage=0
160 useSolverInData=False
161
162 self.hasState=False
163
164 customData=[]
165 for i,c in enumerate(self.opts.customData):
166 lst=c.split("=")
167 if len(lst)==2:
168 name,spec=lst
169 name+="_"
170 elif len(lst)==1:
171 name,spec="Custom%d" % (i+1),c
172 else:
173 self.error("Custom specification",c,"does not fit the pattern 'name=subs1::subs2::..'")
174 customData.append((name,spec.split("::")))
175
176 if len(customData)>0 and not self.opts.solverNameForCustom:
177 self.warning("Parameter '--solver-name-for-custom-data' should be set if '--custom-data' is used")
178 useSolverInData=True
179
180 for d in dirs:
181 for n in listdir(d):
182 cName=path.join(d,n)
183 if path.isdir(cName):
184 try:
185 sol=SolutionDirectory(cName,archive=None,paraviewLink=False)
186 if sol.isValid():
187 if self.opts.progress:
188 print_("Processing",cName)
189
190 data={}
191
192 data["mtime"]=stat(cName)[ST_MTIME]
193 times=sol.getTimes()
194 try:
195 data["first"]=times[0]
196 except IndexError:
197 data["first"]="None"
198 try:
199 data["last"]=times[-1]
200 except IndexError:
201 data["last"]="None"
202 data["nrSteps"]=len(times)
203 data["procs"]=sol.nrProcs()
204 data["pFirst"]=-1
205 data["pLast"]=-1
206 data["nrParallel"]=-1
207 if self.opts.parallel:
208 pTimes=sol.getParallelTimes()
209 data["nrParallel"]=len(pTimes)
210 if len(pTimes)>0:
211 data["pFirst"]=pTimes[0]
212 data["pLast"]=pTimes[-1]
213 data["name"]=cName
214 data["diskusage"]=-1
215 if self.opts.diskusage:
216 try:
217 data["diskusage"]=int(
218 subprocess.Popen(
219 ["du","-sb",cName],
220 stdout=subprocess.PIPE,
221 stderr=open(os.devnull,"w")
222 ).communicate()[0].split()[0])
223 except IndexError:
224
225 data["diskusage"]=int(
226 subprocess.Popen(
227 ["du","-sk",cName],
228 stdout=subprocess.PIPE
229 ).communicate()[0].split()[0])*1024
230
231 totalDiskusage+=data["diskusage"]
232 if self.opts.parallel:
233 for f in listdir(cName):
234 if re.compile("processor[0-9]+").match(f):
235 data["mtime"]=max(stat(path.join(cName,f))[ST_MTIME],data["mtime"])
236
237 if self.opts.state:
238 try:
239 data["nowTime"]=float(self.readState(sol,"CurrentTime"))
240 except ValueError:
241 data["nowTime"]=None
242
243 try:
244 data["lastOutput"]=time.mktime(time.strptime(self.readState(sol,"LastOutputSeen")))
245 except ValueError:
246 data["lastOutput"]="nix"
247
248 data["state"]=self.readState(sol,"TheState")
249
250 if self.opts.state or self.opts.estimateEndTime:
251 try:
252 data["startedAt"]=time.mktime(time.strptime(self.readState(sol,"StartedAt")))
253 except ValueError:
254 data["startedAt"]="nix"
255
256 if self.opts.startEndTime or self.opts.estimateEndTime:
257 try:
258 ctrlDict=ParsedParameterFile(sol.controlDict(),doMacroExpansion=True)
259 except PyFoamParserError:
260
261 ctrlDict=ParsedParameterFile(sol.controlDict())
262
263 data["startTime"]=ctrlDict["startTime"]
264 data["endTime"]=ctrlDict["endTime"]
265
266 if self.opts.estimateEndTime:
267 data["endTimeEstimate"]=None
268 if self.readState(sol,"TheState")=="Running":
269 gone=time.time()-data["startedAt"]
270 try:
271 current=float(self.readState(sol,"CurrentTime"))
272 frac=(current-data["startTime"])/(data["endTime"]-data["startTime"])
273 except ValueError:
274 frac=0
275 if frac>0:
276 data["endTimeEstimate"]=data["startedAt"]+gone/frac
277
278 if len(customData)>0:
279 fn=None
280 if useSolverInData:
281 data["solver"]="none found"
282
283 for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
284 dirAndTime=[]
285 for g in glob(path.join(cName,"*.analyzed")):
286 pName=path.join(g,f)
287 base=path.basename(g)
288 if base.find("PyFoamRunner.")==0:
289 solverName=base[len("PyFoamRunner."):-len(".analyzed")]
290 else:
291 solverName=None
292 if path.exists(pName):
293 dirAndTime.append((path.getmtime(pName),solverName,pName))
294 dirAndTime.sort(cmp=lambda x,y:cmp(x[0],y[0]))
295 if len(dirAndTime)>0:
296 data["solver"]=dirAndTime[-1][1]
297 pickleFile=dirAndTime[-1][2]
298 break
299
300 solverName=data["solver"]
301 else:
302 solverName=self.opts.solverNameForCustom
303 pickleFile=None
304
305 if pickleFile:
306 fn=pickleFile
307 else:
308 for f in ["pickledData","pickledUnfinishedData","pickledStartData"]:
309 fp=path.join(cName,"PyFoamRunner."+solverName+".analyzed",f)
310 if path.exists(fp):
311 fn=fp
312 break
313 if fn:
314 raw=pickle.Unpickler(open(fn)).load()
315 for n,spec in customData:
316 dt=raw
317 for k in spec:
318 try:
319 dt=dt[k]
320 except KeyError:
321 dt="No key '"+k+"'"
322 break
323 if isinstance(dt,string_types):
324 break
325 data[n]=dt
326 else:
327 for n,spec in customData:
328 data[n]="no file"
329
330 cData.append(data)
331 except OSError:
332 print_(cName,"is unreadable")
333
334 if self.opts.progress:
335 print_("Sorting data")
336
337 if self.opts.reverse:
338 cData.sort(lambda x,y:cmp(y[self.opts.sort],x[self.opts.sort]))
339 else:
340 cData.sort(lambda x,y:cmp(x[self.opts.sort],y[self.opts.sort]))
341
342 if len(cData)==0:
343 print_("No cases found")
344 return
345
346 if self.opts.dump:
347 print_(cData)
348 return
349
350 lens={}
351 for k in list(cData[0].keys()):
352 lens[k]=len(k)
353 for c in cData:
354 for k in ["mtime","lastOutput","startedAt","endTimeEstimate"]:
355 try:
356 if c[k]!=None:
357 if self.opts.relativeTime:
358 c[k]=datetime.timedelta(seconds=long(time.time()-c[k]))
359 else:
360 c[k]=time.asctime(time.localtime(c[k]))
361 except KeyError:
362 pass
363 except TypeError:
364 c[k]=None
365
366 try:
367 c["diskusage"]=humanReadableSize(c["diskusage"])
368 except KeyError:
369 pass
370
371 for k,v in iteritems(c):
372 lens[k]=max(lens[k],len(str(v)))
373
374 format=""
375 spec=["mtime"," | ","first"," - ","last"," (","nrSteps",") "]
376 if self.opts.parallel:
377 spec+=["| ","procs"," : ","pFirst"," - ","pLast"," (","nrParallel",") | "]
378 if self.opts.diskusage:
379 spec+=["diskusage"," | "]
380 if self.hasState:
381 spec+=["nowTime"," s ","state"," | "]
382 if self.opts.advancedState:
383 spec+=["lastOutput"," | ","startedAt"," | "]
384 if self.opts.estimateEndTime:
385 if not self.opts.advancedState:
386 spec+=["startedAt"," | "]
387 spec+=["endTimeEstimate"," | "]
388 if self.opts.startEndTime:
389 spec+=["startTime"," | ","endTime"," | "]
390
391 if useSolverInData:
392 spec+=["solver"," | "]
393 for n,s in customData:
394 spec+=[n," | "]
395
396 spec+=["name"]
397
398 for i,l in enumerate(spec):
399 if not l in list(cData[0].keys()):
400 format+=l
401 else:
402 if i<len(spec)-1:
403 format+="%%(%s)%ds" % (l,lens[l])
404 else:
405 format+="%%(%s)s" % (l)
406
407 if self.opts.progress:
408 print_("Printing\n\n")
409
410 header=format % dict(list(zip(list(cData[0].keys()),list(cData[0].keys()))))
411 print_(header)
412 print_("-"*len(header))
413
414 for d in cData:
415 for k in list(d.keys()):
416 d[k]=str(d[k])
417 print_(format % d)
418
419 if self.opts.diskusage:
420 print_("Total disk-usage:",humanReadableSize(totalDiskusage))
421
422
423
424