1 """
2 Application-class that implements pyFoamCompressCaseFiles.py
3 """
4 from optparse import OptionGroup
5 from os import path,listdir,stat
6 import time,datetime
7 from stat import ST_MTIME
8 import string
9 import subprocess
10 import re,sys
11 from glob import glob
12
13 from .PyFoamApplication import PyFoamApplication
14
15 from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory
16
17 from PyFoam import configuration
18
19 from PyFoam.ThirdParty.six import print_,iteritems,PY3
20
21 from PyFoam.Basics.Utilities import humanReadableSize
22
23 if PY3:
24 long=int
25
28 description="""\
29 Gets a number of directories. If these are OpenFOAM-cases then it goes through them
30 and checks for large uncompressed files and gnuzips them
31 """
32 PyFoamApplication.__init__(self,
33 args=args,
34 description=description,
35 usage="%prog [<directories>]",
36 interspersed=True,
37 changeVersion=False,
38 nr=1,
39 exactNr=False)
40
42 compress=OptionGroup(self.parser,
43 "Compression",
44 "Define what will be compressed")
45 self.parser.add_option_group(compress)
46
47 compress.add_option("--recursive",
48 action="store_true",
49 dest="recursive",
50 default=False,
51 help="Use the specified directories as a starting point and recursively search for cases")
52
53 compress.add_option("--big-size",
54 action="store",
55 type="int",
56 dest="bigSize",
57 default=4095,
58 help="Files with less bytes than this will not be compressed. Default: %default")
59
60 compress.add_option("--compressed-extensions",
61 action="append",
62 dest="extensions",
63 default=["gz","zip","tgz"],
64 help="File extensions for which we assume that they are already compressed. Default: %default")
65
66 compress.add_option("--logfiles",
67 action="store_true",
68 dest="logfile",
69 default=False,
70 help="Compress files in the case directory that end with .logfile (Assuming that these are logfiles generated by PyFoam)")
71
72 feedback=OptionGroup(self.parser,
73 "Feedback",
74 "What should be printed")
75 self.parser.add_option_group(feedback)
76 feedback.add_option("--verbose",
77 action="count",
78 dest="verbose",
79 default=1,
80 help="Print names of the directories processed. Use multiple times for higher verbosity")
81 feedback.add_option("--no-statistics",
82 action="store_false",
83 dest="statistics",
84 default=True,
85 help="Do not print a summary in the end")
86 feedback.add_option("--silent",
87 action="store_true",
88 dest="silent",
89 default=False,
90 help="Switch off all output except warnings")
91
92
94 if self.verbose>1:
95 print_(" Compressing",fName)
96 zippedName=fName+".gz"
97 if path.exists(zippedName):
98 self.warning("Zipped file",zippedName,"already existing for",fName)
99 return
100 oldSize=path.getsize(fName)
101 if oldSize<self.bigSize:
102 if self.verbose>2:
103 print_(" Skipping because it is too small")
104 self.nrSkipped+=1
105 return
106
107
108 ret=subprocess.call(["gzip",fName])
109 if ret!=0 or not path.exists(zippedName) or path.exists(fName):
110 self.warning("Problem compressing file",fName)
111 self.nrProblems+=1
112 return
113 newSize=path.getsize(zippedName)
114 if newSize>oldSize:
115 self.warning("Compression of",fName,"increased the filesize. Old:",
116 humanReadableSize(oldSize),"New:",humanReadableSize(newSize))
117
118 if self.verbose>2:
119 print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize))
120
121 self.nrFiles+=1
122 self.prevSize+=oldSize
123 self.nowSize+=newSize
124
137
139 if not path.exists(dirName):
140 self.error("Directory",dirName,"does not exist")
141 s=SolutionDirectory(dirName,
142 archive=None,
143 paraviewLink=False,
144 parallel=True,
145 tolerant=True)
146 if not s.isValid():
147 if warn:
148 print_("Directory",dirName,"is not an OpenFOAM-case")
149 return
150
151 self.nrDir+=1
152 oldNr=self.nrFiles
153 oldUnc=self.prevSize
154 oldCon=self.nowSize
155
156 if self.verbose>0:
157 print_("Processing case",dirName)
158
159
160 for d in glob(path.join(dirName,"*","polyMesh"))+glob(path.join(dirName,"*","*","polyMesh")):
161 if path.isdir(d):
162 self.compressDirectory(d)
163
164
165 for t in s:
166 self.compressDirectory(t.name)
167
168
169 if self.opts.logfile:
170 for f in glob(path.join(dirName,"*.logfile")):
171 self.compressFile(path.join(dirName,f))
172
173
174 for p in s.procDirs:
175 self.compressDirectory(path.join(dirName,p))
176 if self.nrFiles>oldNr and self.verbose>0:
177 print_(" -> ",self.nrFiles-oldNr,"files compressed.",
178 humanReadableSize((self.prevSize-oldUnc)-(self.nowSize-oldCon)),"gained")
179
181 if self.verbose>1:
182 print_("Recursively checking",dirName)
183 if path.isdir(dirName):
184 s=SolutionDirectory(dirName,archive=None,paraviewLink=False,parallel=True)
185 if s.isValid():
186 try:
187 self.compressCase(dirName)
188 except OSError:
189 e = sys.exc_info()[1]
190 self.warning("Problem processing",dirName,":",e)
191 return
192
193 for f in listdir(dirName):
194 name=path.join(dirName,f)
195 try:
196 if path.isdir(name):
197 self.recursiveCompress(name)
198 except OSError:
199 e = sys.exc_info()[1]
200 self.warning("Problem processing",name,":",e)
201
203 dirs=self.parser.getArgs()
204
205 self.bigSize=self.opts.bigSize
206 self.verbose=self.opts.verbose
207 self.extensions=["."+e.lower() for e in self.opts.extensions]
208
209 if self.opts.silent:
210 self.verbose=0
211 self.opts.statistics=False
212
213
214 self.nrDir=0
215 self.nrCompressed=0
216 self.nrFiles=0
217 self.prevSize=0
218 self.nowSize=0
219 self.nrSkipped=0
220 self.nrProblems=0
221
222 try:
223 for d in dirs:
224 p=path.abspath(d)
225 if self.opts.recursive:
226 self.recursiveCompress(p)
227 else:
228 self.compressCase(p,warn=True)
229 except KeyboardInterrupt:
230 print_("Rudely interrupted by Control-C")
231
232 if self.opts.statistics:
233 if self.verbose>0:
234 print_()
235 print_(self.nrDir,"case directories processed")
236 if self.nrFiles==0:
237 print_("No files to compress found")
238 else:
239 print_(self.nrFiles,"files processed")
240 print_("Reduced total size from",humanReadableSize(self.prevSize),
241 "to",humanReadableSize(self.nowSize),
242 ". This is",(100.*self.nowSize)/self.prevSize,"percent of the original")
243 if self.nrSkipped>0:
244 print_("Skipped",self.nrSkipped,"files because they were smaller than",self.bigSize)
245 if self.nrCompressed>0:
246 print_("Skipped",self.nrCompressed,"files because they are already compressed")
247 if self.nrProblems>0:
248 print_("Problems during the compression of",self.nrProblems,"files")
249
250
251