1 """
2 Application-class that implements pyFoamCompressCaseFiles.py
3 """
4 from optparse import OptionGroup
5 from os import path,listdir
6 import subprocess
7 import sys
8 from glob import glob
9
10 from .PyFoamApplication import PyFoamApplication
11
12 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
13
14 from PyFoam.ThirdParty.six import print_,PY3
15
16 from PyFoam.Basics.Utilities import humanReadableSize
17
18 if PY3:
19 long=int
20
22 - def __init__(self,
23 args=None,
24 **kwargs):
25 description="""\
26 Gets a number of directories. If these are OpenFOAM-cases then it goes through them
27 and checks for large uncompressed files and gnuzips them
28 """
29 PyFoamApplication.__init__(self,
30 args=args,
31 description=description,
32 usage="%prog [<directories>]",
33 interspersed=True,
34 changeVersion=False,
35 nr=1,
36 exactNr=False,
37 **kwargs)
38
40 compress=OptionGroup(self.parser,
41 "Compression",
42 "Define what will be compressed")
43 self.parser.add_option_group(compress)
44
45 compress.add_option("--recursive",
46 action="store_true",
47 dest="recursive",
48 default=False,
49 help="Use the specified directories as a starting point and recursively search for cases")
50
51 compress.add_option("--big-size",
52 action="store",
53 type="int",
54 dest="bigSize",
55 default=4095,
56 help="Files with less bytes than this will not be compressed. Default: %default")
57
58 compress.add_option("--compressed-extensions",
59 action="append",
60 dest="extensions",
61 default=["gz","zip","tgz"],
62 help="File extensions for which we assume that they are already compressed. Default: %default")
63
64 compress.add_option("--logfiles",
65 action="store_true",
66 dest="logfile",
67 default=False,
68 help="Compress files in the case directory that end with .logfile (Assuming that these are logfiles generated by PyFoam)")
69
70 feedback=OptionGroup(self.parser,
71 "Feedback",
72 "What should be printed")
73 self.parser.add_option_group(feedback)
74 feedback.add_option("--verbose",
75 action="count",
76 dest="verbose",
77 default=1,
78 help="Print names of the directories processed. Use multiple times for higher verbosity")
79 feedback.add_option("--no-statistics",
80 action="store_false",
81 dest="statistics",
82 default=True,
83 help="Do not print a summary in the end")
84 feedback.add_option("--silent",
85 action="store_true",
86 dest="silent",
87 default=False,
88 help="Switch off all output except warnings")
89
90
92 if self.verbose>1:
93 print_(" Compressing",fName)
94 zippedName=fName+".gz"
95 if path.exists(zippedName):
96 self.warning("Zipped file",zippedName,"already existing for",fName)
97 return
98 oldSize=path.getsize(fName)
99 if oldSize<self.bigSize:
100 if self.verbose>2:
101 print_(" Skipping because it is too small")
102 self.nrSkipped+=1
103 return
104
105
106 ret=subprocess.call(["gzip",fName])
107 if ret!=0 or not path.exists(zippedName) or path.exists(fName):
108 self.warning("Problem compressing file",fName)
109 self.nrProblems+=1
110 return
111 newSize=path.getsize(zippedName)
112 if newSize>oldSize:
113 self.warning("Compression of",fName,"increased the filesize. Old:",
114 humanReadableSize(oldSize),"New:",humanReadableSize(newSize))
115
116 if self.verbose>2:
117 print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize))
118
119 self.nrFiles+=1
120 self.prevSize+=oldSize
121 self.nowSize+=newSize
122
135
137 if not path.exists(dirName):
138 self.error("Directory",dirName,"does not exist")
139 s=SolutionDirectory(dirName,
140 archive=None,
141 paraviewLink=False,
142 parallel=True,
143 tolerant=True)
144 if not s.isValid():
145 if warn:
146 print_("Directory",dirName,"is not an OpenFOAM-case")
147 return
148
149 self.nrDir+=1
150 oldNr=self.nrFiles
151 oldUnc=self.prevSize
152 oldCon=self.nowSize
153
154 if self.verbose>0:
155 print_("Processing case",dirName)
156
157
158 for d in glob(path.join(dirName,"*","polyMesh"))+glob(path.join(dirName,"*","*","polyMesh")):
159 if path.isdir(d):
160 self.compressDirectory(d)
161
162
163 for t in s:
164 self.compressDirectory(t.name)
165
166
167 if self.opts.logfile:
168 for f in glob(path.join(dirName,"*.logfile")):
169 self.compressFile(path.join(dirName,f))
170
171
172 for p in s.procDirs:
173 self.compressDirectory(path.join(dirName,p))
174 if self.nrFiles>oldNr and self.verbose>0:
175 print_(" -> ",self.nrFiles-oldNr,"files compressed.",
176 humanReadableSize((self.prevSize-oldUnc)-(self.nowSize-oldCon)),"gained")
177
179 if self.verbose>1:
180 print_("Recursively checking",dirName)
181 if path.isdir(dirName):
182 s=SolutionDirectory(dirName,archive=None,paraviewLink=False,parallel=True)
183 if s.isValid():
184 try:
185 self.compressCase(dirName)
186 except OSError:
187 e = sys.exc_info()[1]
188 self.warning("Problem processing",dirName,":",e)
189 return
190
191 for f in listdir(dirName):
192 name=path.join(dirName,f)
193 try:
194 if path.isdir(name):
195 self.recursiveCompress(name)
196 except OSError:
197 e = sys.exc_info()[1]
198 self.warning("Problem processing",name,":",e)
199
201 dirs=self.parser.getArgs()
202
203 self.bigSize=self.opts.bigSize
204 self.verbose=self.opts.verbose
205 self.extensions=["."+e.lower() for e in self.opts.extensions]
206
207 if self.opts.silent:
208 self.verbose=0
209 self.opts.statistics=False
210
211
212 self.nrDir=0
213 self.nrCompressed=0
214 self.nrFiles=0
215 self.prevSize=0
216 self.nowSize=0
217 self.nrSkipped=0
218 self.nrProblems=0
219
220 try:
221 for d in dirs:
222 p=path.abspath(d)
223 if self.opts.recursive:
224 self.recursiveCompress(p)
225 else:
226 self.compressCase(p,warn=True)
227 except KeyboardInterrupt:
228 print_("Rudely interrupted by Control-C")
229
230 if self.opts.statistics:
231 if self.verbose>0:
232 print_()
233 print_(self.nrDir,"case directories processed")
234 if self.nrFiles==0:
235 print_("No files to compress found")
236 else:
237 print_(self.nrFiles,"files processed")
238 print_("Reduced total size from",humanReadableSize(self.prevSize),
239 "to",humanReadableSize(self.nowSize),
240 ". This is",(100.*self.nowSize)/self.prevSize,"percent of the original")
241 if self.nrSkipped>0:
242 print_("Skipped",self.nrSkipped,"files because they were smaller than",self.bigSize)
243 if self.nrCompressed>0:
244 print_("Skipped",self.nrCompressed,"files because they are already compressed")
245 if self.nrProblems>0:
246 print_("Problems during the compression of",self.nrProblems,"files")
247
248
249