1 """
2 Application-class that implements pyFoamIPythonNotebook.py
3 """
4 from optparse import OptionGroup
5
6 from .PyFoamApplication import PyFoamApplication
7 from PyFoam.IPythonHelpers.Notebook import Notebook
8 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
9 from PyFoam.Basics.FoamOptionParser import Subcommand
10
11 from os import path
12 import sys,re
13
14 from PyFoam.ThirdParty.six import print_,u
15
17 - def __init__(self,
18 args=None,
19 **kwargs):
20 description="""\
21 This utility creates and manipulates IPython-Notebooks that are related to
22 OpenFOAM-cases. The Notebooks are only used as a start for the own evaluations of the user
23 """
24 PyFoamApplication.__init__(self,
25 args=args,
26 description=description,
27 usage="%prog COMMAND [<arguments>]",
28 changeVersion=False,
29 subcommands=True,
30 **kwargs)
31
33
34 createCmd=Subcommand(name='create',
35 help="Create a new IPython-notebook for a case",
36 aliases=("new","mk",),
37 nr=1,
38 exactNr=True)
39 self.parser.addSubcommand(createCmd,
40 usage="%prog COMMAND <caseDirectory>")
41
42 copyCmd=Subcommand(name='copy',
43 help="Gets an existing notebook and rewrites it to fit a new case (this assumes that the original notebook was built with this utility)",
44 aliases=("cp",),
45 nr=2,
46 exactNr=True)
47 self.parser.addSubcommand(copyCmd,
48 usage="%prog COMMAND <originalNotebook> <caseDirectory>")
49
50 infoCmd=Subcommand(name='info',
51 help="Check whether an IPython-Notebook is created by this Utility and print info",
52 aliases=("report",),
53 nr=1,
54 exactNr=False)
55 self.parser.addSubcommand(infoCmd,
56 usage="%prog COMMAND <notebookFile> [<more notebook files>]")
57
58 cleanCmd=Subcommand(name='clean',
59 help="Remove unneeded cells from the notebook",
60 aliases=("purge",),
61 nr=1,
62 exactNr=True)
63 self.parser.addSubcommand(cleanCmd,
64 usage="%prog COMMAND <notebookFile>")
65
66
67 for cmd in [copyCmd,createCmd]:
68 outOpts=OptionGroup(cmd.parser,
69 "Write Options",
70 "Where the Notebook should be created")
71 outOpts.add_option("--force-write",
72 action="store_true",
73 dest="forceWrite",
74 default=False,
75 help="Force writing if the file already exists")
76 outOpts.add_option("--destination-file",
77 action="store",
78 dest="destinationFile",
79 default=None,
80 help="Write to this filename. If unset the notebook is written to the case it is created for as <casename>.ipynb. If the destination is directory the file is created in this directory as <casename>.ipynb. Otherwise the fie is created according to specification")
81 outOpts.add_option("--relative-path",
82 action="store_false",
83 dest="absolutePath",
84 default=True,
85 help="Keep the relative path to the directory as specified by the user. Otherwise the path is rewritten as an absolute path")
86 outOpts.add_option("--case-variable-name",
87 action="store",
88 dest="caseVariable",
89 default="case",
90 help="Name of the variable representing the case in the notebook. Defaut: %default")
91 cmd.parser.add_option_group(outOpts)
92
93 for cmd in [cleanCmd]:
94 cleanOpts=OptionGroup(cmd.parser,
95 "Clean Options",
96 "What should be cleaned")
97 cleanOpts.add_option("--keep-selector",
98 action="store_false",
99 dest="cleanSelector",
100 default=True,
101 help="Keep the data selectors")
102 cleanOpts.add_option("--keep-developer",
103 action="store_false",
104 dest="cleanDeveloper",
105 default=True,
106 help="Clean out the developer stuff")
107 cleanOpts.add_option("--clean-comments",
108 action="store_true",
109 dest="cleanComment",
110 default=False,
111 help="Clean out the comments created by this utility")
112 cleanOpts.add_option("--clean-headings",
113 action="store_true",
114 dest="cleanHeading",
115 default=False,
116 help="Clean out the headings created by this utility")
117 cleanOpts.add_option("--clean-report",
118 action="store_true",
119 dest="cleanReport",
120 default=False,
121 help="Clean out the case report created by this utility")
122 cleanOpts.add_option("--clean-info",
123 action="store_true",
124 dest="cleanInfo",
125 default=False,
126 help="Clean out information statements")
127 cleanOpts.add_option("--clean-output",
128 action="store_true",
129 dest="cleanOutput",
130 default=False,
131 help="Strip out the output cells (results)")
132 cleanOpts.add_option("--clean-custom-tag",
133 action="append",
134 dest="customTags",
135 default=[],
136 help="Clean cells tagged with this custom tag. Can be specified more than once")
137 cmd.parser.add_option_group(cleanOpts)
138
139 outOpts=OptionGroup(cmd.parser,
140 "Write Options",
141 "How the cleaned notebook should be written")
142 outOpts.add_option("--overwrite",
143 action="store_true",
144 dest="overwrite",
145 default=False,
146 help="Overwrite the old notebook")
147 outOpts.add_option("--outfile",
148 action="store",
149 dest="outfile",
150 default=None,
151 help="Write to a new notebook here")
152 outOpts.add_option("--force",
153 action="store_true",
154 dest="force",
155 default=False,
156 help="If the outfile already exists overwrite it")
157 cmd.parser.add_option_group(outOpts)
158
159 for cmd in [createCmd]:
160 contentOpts=OptionGroup(cmd.parser,
161 "Content Options",
162 "What should be added to the notebook")
163 contentOpts.add_option("--no-case-report",
164 action="store_false",
165 dest="caseReport",
166 default=True,
167 help="Do not give a general overview of the case")
168 contentOpts.add_option("--no-additional-imports",
169 action="store_false",
170 dest="additional",
171 default=True,
172 help="Do not import packages that make the notebook neater")
173 contentOpts.add_option("--long-boundary-conditions",
174 action="store_true",
175 dest="longBCs",
176 default=False,
177 help="Long boundary conditions")
178 contentOpts.add_option("--no-parallel-report",
179 action="store_false",
180 dest="parallelReport",
181 default=True,
182 help="Do not report about parallelization")
183 contentOpts.add_option("--no-postprocessing",
184 action="store_false",
185 dest="postprocessing",
186 default=True,
187 help="Do not report about available postprocessing data")
188 contentOpts.add_option("--no-data-selectors",
189 action="store_false",
190 dest="selectors",
191 default=True,
192 help="Do not add data selectors for the available postprocessing data")
193 cmd.parser.add_option_group(contentOpts)
194
196 if self.cmdname in ["create","copy"]:
197 if self.cmdname=="create":
198 dest=self.parser.getArgs()[0]
199 else:
200 dest=self.parser.getArgs()[1]
201 sol=SolutionDirectory(dest,
202 paraviewLink=False,
203 archive=None)
204 fName=path.join(sol.name,path.basename(sol.name)+".ipynb")
205 if self.opts.destinationFile:
206 fName=self.opts.destinationFile
207 if path.isdir(fName):
208 fName=path.join(fName,path.basename(sol.name))
209 if path.splitext(fName)[1]!=".ipynb":
210 fName+=".ipynb"
211 if self.opts.absolutePath:
212 usedDest=sol.name
213 else:
214 usedDest=path.relpath(sol.name,
215 start=path.dirname(path.abspath(
216 fName)))
217 if path.exists(fName):
218 if not self.opts.forceWrite:
219 self.error("File",fName,"already existing")
220 else:
221 self.warning("Overwriting",fName)
222 nb=Notebook(name=path.basename(sol.name))
223 nb.pyFoamMetaData()["description"]="Created by "+self.parser.get_prog_name()
224 if self.cmdname=="create":
225 nb.addHeading("Imports and administrative stuff",
226 level=1,classes="heading")
227 if self.opts.developerMode:
228 nb.addMarkdown("This part only needed by developers (reload imports)",
229 classes=("comment","developer"))
230 nb.addCode("%load_ext autoreload",classes="developer")
231 nb.addCode("%autoreload 2",classes="developer")
232 nb.addMarkdown("Make sure that plots are inlined",
233 classes="comment")
234 nb.addCode("%matplotlib inline")
235 if self.opts.additional:
236 nb.addHeading("Additional imports for convenience",
237 level=2,classes=("heading","additional"))
238 nb.addMarkdown("Allow panning and zooming in plots. Slower than regular plotting so for big data you might want to use `mpld3.disable_notebook()` and erase this cell.",
239 classes=("comment","additional"))
240 nb.addCode(
241 """try:
242 import mpld3
243 mpld3.enable_notebook()
244 except ImportError:
245 print 'No mpld3-library. No interactive plots'""",classes="additional")
246 nb.addMarkdown(
247 """Uncomment this code to change the size of the plots""")
248 nb.addCode(
249 """# import matplotlib.pylab as pylab
250 # pylab.rcParams["figure.figsize"]=(12,8)""")
251 nb.addMarkdown(
252 """Wrapper with additional functionality to the regular Pandas-`DataFrame`:
253
254 * `addData()` for adding columns from other data sets (with resampling
255 * `integrals()` and `weightedAverage()`. Also extended `descripe()` that returns this data
256
257 Most Pandas-operations (like slicing) will return a Pandas-`DataFrame`. By enclosing this in `DataFrame(...)` you can 'add' this functionality to your data. PyFoam operations return this extended `DataFrame` automatically""",
258 classes=("comment","additional"))
259 nb.addCode("from PyFoam.Wrappers.Pandas import PyFoamDataFrame as DataFrame",classes="additional")
260 nb.addHeading("Data storage",
261 level=2,classes=("heading"))
262 nb.addMarkdown("This is the support for permanently storing data into the notebook",
263 classes="comment")
264 nb.addCode("from PyFoam.IPythonHelpers import storage")
265 nb.addMarkdown("Due to technical problems the next line has to be executed 'by hand' (it will not work poperly if called from `Run All` or similar). When reopening the page the JavaScript-error is normal (it will go away once the cell is executed). Reading can take some time and the next command will appear to 'hang'",
266 classes="comment")
267 nb.addCode("store=storage()")
268 nb.addMarkdown("The next line switches on the behaviour that items specified with `store(name,func)` will be stored permanently in the notebook. Uncomment if you want this behaviour",
269 classes="comment")
270 nb.addCode("# store.autowriteOn()")
271 nb.addMarkdown("The next line switches off the default behaviour that for items specified with `store(name,func)` if `name` is already specified in the permant storage this value is used and `func` is ignored",
272 classes="comment")
273 nb.addCode("# store.autoreadOff()")
274 nb.addHeading("Case data",
275 level=2,classes=("heading"))
276 nb.addMarkdown("This class makes it easy to access case data. Use tab-completion for available methods",
277 classes="comment")
278 nb.addCode("from PyFoam.IPythonHelpers.Case import Case")
279 nb.addHeading("The Case",classes="heading")
280 v=self.opts.caseVariable
281 nb.addCode("%s=Case('%s')" % (v,usedDest),classes="case",
282 pyFoam={"caseVar":v,"usedDirectory":usedDest,
283 "casePath":sol.name})
284 if self.opts.caseReport:
285 nb.addHeading("Case Report",level=2,
286 classes=("report","heading"))
287 regions=sorted(sol.getRegions(defaultRegion=True))
288 namedRegions=[r for r in regions if r!=None]
289 if len(namedRegions)>0:
290 nb.addMarkdown("Contains named regions *"+
291 ", ".join(namedRegions)+"*",
292 classes=("info","report"))
293 if sol.procNr>0:
294 nb.addMarkdown("Case seems to be decomposed to "+
295 str(sol.procNr)+" processors",
296 classes=("info","report"))
297 for region in regions:
298 if region==None:
299 level=3
300 regionStr=""
301 else:
302 nb.addHeading("Region "+region,
303 level=3,classes=("heading","report"))
304 level=4
305 regionStr="region='%s'," % region
306 nb.addCode("%s.size(%slevel=%d)" % (v,regionStr,level),
307 classes="report")
308 nb.addCode("%s.boundaryConditions(%slevel=%d)" % (v,regionStr,level),
309 classes="report")
310 nb.addCode("%s.dimensions(%slevel=%d)" % (v,regionStr,level),
311 classes="report")
312 nb.addCode("%s.internalField(%slevel=%d)" % (v,regionStr,level),
313 classes="report")
314 if self.opts.longBCs:
315 nb.addCode("%s.longBoundaryConditions(%slevel=%d)" % (regionStr,v,level),
316 classes="report")
317 if sol.procNr>0 and self.opts.parallelReport:
318 nb.addCode("%s.decomposition(%slevel=%d)" % (v,regionStr,level),
319 classes="report")
320 nb.addCode("%s.processorMatrix(%slevel=%d)" % (v,regionStr,level),
321 classes="report")
322 if self.opts.postprocessing:
323 nb.addHeading("Postprocessing data",classes="heading")
324 if len(sol.timelines)>0:
325 nb.addMarkdown("Timelines",classes="info")
326 nb.addCode("%s.sol.timelines" % v,classes="info")
327 if len(sol.samples)>0:
328 nb.addMarkdown("Samples",classes="info")
329 nb.addCode("%s.sol.samples" % v,classes="info")
330 if len(sol.surfaces)>0:
331 nb.addMarkdown("Surfaces",classes="info")
332 nb.addCode("%s.sol.surfaces" % v,classes="info")
333 if len(sol.distributions)>0:
334 nb.addMarkdown("Distributions",classes="info")
335 nb.addCode("%s.sol.distributions" % v,classes="info")
336 if len(sol.pickledData)>0:
337 nb.addMarkdown("Pickled data files",classes="info")
338 nb.addCode("%s.sol.pickledData" % v,classes="info")
339 if len(sol.pickledPlots)>0:
340 nb.addMarkdown("Pickled plot files",classes="info")
341 nb.addCode("%s.sol.pickledPlots" % v,classes="info")
342 if self.opts.selectors:
343 sel=[("timeline",sol.timelines),
344 ("sample",sol.samples),
345 ("distribution",sol.distributions)]
346 for desc,items in sel:
347 if len(items)>0:
348 nb.addHeading(desc.capitalize()+
349 " selectors",level=3,
350 classes=("heading","selector"))
351 for i in items:
352 nb.addCode("%s.%sSelector('%s')" %
353 (v,desc,i),
354 classes="selector")
355 if len(sol.pickledPlots)>0 or len(sol.pickledData)>0:
356 nb.addHeading("Data selectors",level=3,
357 classes=("heading","selector"))
358 if len(sol.pickledPlots)>0:
359 nb.addCode("%s.pickledPlotSelector()" % v,classes="selector")
360 if len(sol.pickledData)>0:
361 nb.addCode("%s.pickledDataSelector()" % v,classes="selector")
362
363 nb.addHeading("User evaluations",classes="heading")
364 nb.addMarkdown("Now add your own stuff",classes="comment")
365 elif self.cmdname=="copy":
366 src=self.parser.getArgs()[0]
367 nb=Notebook(src)
368 cnt=0
369 for c in nb:
370 if c.isClass("case"):
371 cnt+=1
372 if cnt>1:
373 self.error(src,"has more than one 'case'-cell")
374 py=c.meta()[u("pyFoam")]
375 used=py["usedDirectory"]
376 input=[]
377 changed=False
378 for l in c["input"]:
379 if l.find(used)>=0:
380 input.append(l.replace(used,usedDest))
381 changed=True
382 else:
383 input.append(l)
384 if not changed:
385 self.warning(used,"not found")
386 py["usedDirectory"]=usedDest
387 py["casePath"]=sol.name
388 c["input"]=input
389 else:
390 self.error("Unimplemented:",self.cmdname)
391 nb.writeToFile(fName)
392 elif self.cmdname=="info":
393 for n in self.parser.getArgs():
394 print_(n)
395 print_("-"*len(n))
396 nb=Notebook(n)
397 meta=nb.pyFoamMetaData()
398 try:
399 origin=meta["createdBy"]
400 except KeyError:
401 origin="unknown"
402 try:
403 created=meta["createdTime"]
404 except KeyError:
405 created="unknown"
406 try:
407 created=meta["createdTime"]
408 except KeyError:
409 created="unknown"
410 try:
411 modified=meta["modificationTime"]
412 except KeyError:
413 modified="unknown"
414 print_("Created by",origin,"at",created,
415 "modified",modified)
416 classes={}
417 cases={}
418 nrOutput=0
419 for c in nb:
420 if "outputs" in c:
421 if len(c["outputs"])>0:
422 nrOutput+=1
423 try:
424 py=c.meta()[u("pyFoam")]
425 except KeyError:
426 continue
427 try:
428 cl=py["classes"]
429 for c in cl:
430 try:
431 classes[c]+=1
432 except KeyError:
433 classes[c]=1
434 except KeyError:
435 pass
436 if "caseVar" in py:
437 try:
438 cases[py["caseVar"]]=py["casePath"]
439 except KeyError:
440 pass
441 print_(len(nb),"cells. Classes:",
442 ", ".join([k+":"+str(classes[k]) for k in sorted(classes.keys())]))
443 print_("Cells with output:",nrOutput)
444 print("Case-Variables:")
445 for k in sorted(cases.keys()):
446 print_(" ",k,":",cases[k])
447
448 print_()
449 elif self.cmdname=="clean":
450 nb=Notebook(self.parser.getArgs()[0])
451 if not self.opts.overwrite and not self.opts.outfile:
452 self.error("Either specify --overwrite or --outfile")
453 if self.opts.overwrite and self.opts.outfile:
454 self.error("Only specify --overwrite or --outfile")
455 if self.opts.outfile:
456 if path.exists(self.opts.outfile):
457 if not self.opts.force:
458 self.error("File",self.opts.outfile,"exists")
459 else:
460 self.warning("Overwriting",self.opts.outfile)
461 else:
462 if path.splitext(self.opts.outfile)[1]!=".ipynb":
463 self.warning("Appending '.ipynb' to",self.opts.outfile)
464 self.opts.outfile+=".ipynb"
465 if self.opts.overwrite:
466 toFile=self.parser.getArgs()[0]
467 else:
468 toFile=self.opts.outfile
469
470 removeClasses=self.opts.customTags[:]
471 if self.opts.cleanSelector:
472 removeClasses.append("selector")
473 if self.opts.cleanDeveloper:
474 removeClasses.append("developer")
475 if self.opts.cleanHeading:
476 removeClasses.append("heading")
477 if self.opts.cleanComment:
478 removeClasses.append("comment")
479 if self.opts.cleanReport:
480 removeClasses.append("report")
481 if self.opts.cleanInfo:
482 removeClasses.append("info")
483
484 print_("Cleaning cells tagged with: "+" ".join(sorted(removeClasses)))
485
486 nb.reset([c for c in nb if not c.isClass(removeClasses)])
487 if self.opts.cleanOutput:
488 print_("Removing output")
489 for c in nb:
490 if "outputs" in c:
491 c["outputs"]=[]
492
493 nb.writeToFile(toFile)
494 else:
495 self.error("Unimplemented command",self.cmdname)
496
497
498