Package PyFoam :: Package Applications :: Module CompressCaseFiles
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Applications.CompressCaseFiles

  1  """ 
  2  Application-class that implements pyFoamCompressCaseFiles.py 
  3  """ 
  4  from optparse import OptionGroup 
  5  from os import path,listdir 
  6  import subprocess 
  7  import sys 
  8  from glob import glob 
  9   
 10  from .PyFoamApplication import PyFoamApplication 
 11   
 12  from PyFoam.RunDictionary.SolutionDirectory import SolutionDirectory 
 13   
 14  from PyFoam.ThirdParty.six import print_,PY3 
 15   
 16  from PyFoam.Basics.Utilities import humanReadableSize 
 17   
 18  if PY3: 
 19      long=int 
 20   
21 -class CompressCaseFiles(PyFoamApplication):
22 - def __init__(self, 23 args=None, 24 **kwargs):
25 description="""\ 26 Gets a number of directories. If these are OpenFOAM-cases then it goes through them 27 and checks for large uncompressed files and gnuzips them 28 """ 29 PyFoamApplication.__init__(self, 30 args=args, 31 description=description, 32 usage="%prog [<directories>]", 33 interspersed=True, 34 changeVersion=False, 35 nr=1, 36 exactNr=False, 37 **kwargs)
38
39 - def addOptions(self):
40 compress=OptionGroup(self.parser, 41 "Compression", 42 "Define what will be compressed") 43 self.parser.add_option_group(compress) 44 45 compress.add_option("--recursive", 46 action="store_true", 47 dest="recursive", 48 default=False, 49 help="Use the specified directories as a starting point and recursively search for cases") 50 51 compress.add_option("--big-size", 52 action="store", 53 type="int", 54 dest="bigSize", 55 default=4095, 56 help="Files with less bytes than this will not be compressed. Default: %default") 57 58 compress.add_option("--compressed-extensions", 59 action="append", 60 dest="extensions", 61 default=["gz","zip","tgz"], 62 help="File extensions for which we assume that they are already compressed. Default: %default") 63 64 compress.add_option("--logfiles", 65 action="store_true", 66 dest="logfile", 67 default=False, 68 help="Compress files in the case directory that end with .logfile (Assuming that these are logfiles generated by PyFoam)") 69 70 feedback=OptionGroup(self.parser, 71 "Feedback", 72 "What should be printed") 73 self.parser.add_option_group(feedback) 74 feedback.add_option("--verbose", 75 action="count", 76 dest="verbose", 77 default=1, 78 help="Print names of the directories processed. Use multiple times for higher verbosity") 79 feedback.add_option("--no-statistics", 80 action="store_false", 81 dest="statistics", 82 default=True, 83 help="Do not print a summary in the end") 84 feedback.add_option("--silent", 85 action="store_true", 86 dest="silent", 87 default=False, 88 help="Switch off all output except warnings")
89 90
91 - def compressFile(self,fName):
92 if self.verbose>1: 93 print_(" Compressing",fName) 94 zippedName=fName+".gz" 95 if path.exists(zippedName): 96 self.warning("Zipped file",zippedName,"already existing for",fName) 97 return 98 oldSize=path.getsize(fName) 99 if oldSize<self.bigSize: 100 if self.verbose>2: 101 print_(" Skipping because it is too small") 102 self.nrSkipped+=1 103 return 104 105 # use gzip because that way the responsibility of removing the old file is with a 'tried and testd' program 106 ret=subprocess.call(["gzip",fName]) 107 if ret!=0 or not path.exists(zippedName) or path.exists(fName): 108 self.warning("Problem compressing file",fName) 109 self.nrProblems+=1 110 return 111 newSize=path.getsize(zippedName) 112 if newSize>oldSize: 113 self.warning("Compression of",fName,"increased the filesize. Old:", 114 humanReadableSize(oldSize),"New:",humanReadableSize(newSize)) 115 116 if self.verbose>2: 117 print_(" Old size:",humanReadableSize(oldSize),"New size:",humanReadableSize(newSize)) 118 119 self.nrFiles+=1 120 self.prevSize+=oldSize 121 self.nowSize+=newSize
122
123 - def compressDirectory(self,dirName):
124 if self.verbose>1: 125 print_(" Checking",dirName,"for compressible files") 126 for f in listdir(dirName): 127 if path.isdir(path.join(dirName,f)): 128 self.compressDirectory(path.join(dirName,f)) 129 else: 130 name,ext=path.splitext(f) 131 if ext.lower() not in self.extensions: 132 self.compressFile(path.join(dirName,f)) 133 else: 134 self.nrCompressed+=1
135
136 - def compressCase(self,dirName,warn=False):
137 if not path.exists(dirName): 138 self.error("Directory",dirName,"does not exist") 139 s=SolutionDirectory(dirName, 140 archive=None, 141 paraviewLink=False, 142 parallel=True, 143 tolerant=True) 144 if not s.isValid(): 145 if warn: 146 print_("Directory",dirName,"is not an OpenFOAM-case") 147 return 148 149 self.nrDir+=1 150 oldNr=self.nrFiles 151 oldUnc=self.prevSize 152 oldCon=self.nowSize 153 154 if self.verbose>0: 155 print_("Processing case",dirName) 156 157 # compress meshes 158 for d in glob(path.join(dirName,"*","polyMesh"))+glob(path.join(dirName,"*","*","polyMesh")): 159 if path.isdir(d): 160 self.compressDirectory(d) 161 162 # compress times 163 for t in s: 164 self.compressDirectory(t.name) 165 166 # compress logfiles if requested 167 if self.opts.logfile: 168 for f in glob(path.join(dirName,"*.logfile")): 169 self.compressFile(path.join(dirName,f)) 170 171 # processor direcories 172 for p in s.procDirs: 173 self.compressDirectory(path.join(dirName,p)) 174 if self.nrFiles>oldNr and self.verbose>0: 175 print_(" -> ",self.nrFiles-oldNr,"files compressed.", 176 humanReadableSize((self.prevSize-oldUnc)-(self.nowSize-oldCon)),"gained")
177
178 - def recursiveCompress(self,dirName):
179 if self.verbose>1: 180 print_("Recursively checking",dirName) 181 if path.isdir(dirName): 182 s=SolutionDirectory(dirName,archive=None,paraviewLink=False,parallel=True) 183 if s.isValid(): 184 try: 185 self.compressCase(dirName) 186 except OSError: 187 e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' 188 self.warning("Problem processing",dirName,":",e) 189 return 190 191 for f in listdir(dirName): 192 name=path.join(dirName,f) 193 try: 194 if path.isdir(name): 195 self.recursiveCompress(name) 196 except OSError: 197 e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' 198 self.warning("Problem processing",name,":",e)
199
200 - def run(self):
201 dirs=self.parser.getArgs() 202 203 self.bigSize=self.opts.bigSize 204 self.verbose=self.opts.verbose 205 self.extensions=["."+e.lower() for e in self.opts.extensions] 206 207 if self.opts.silent: 208 self.verbose=0 209 self.opts.statistics=False 210 211 # for the statistics 212 self.nrDir=0 213 self.nrCompressed=0 214 self.nrFiles=0 215 self.prevSize=0 216 self.nowSize=0 217 self.nrSkipped=0 218 self.nrProblems=0 219 220 try: 221 for d in dirs: 222 p=path.abspath(d) 223 if self.opts.recursive: 224 self.recursiveCompress(p) 225 else: 226 self.compressCase(p,warn=True) 227 except KeyboardInterrupt: 228 print_("Rudely interrupted by Control-C") 229 230 if self.opts.statistics: 231 if self.verbose>0: 232 print_() 233 print_(self.nrDir,"case directories processed") 234 if self.nrFiles==0: 235 print_("No files to compress found") 236 else: 237 print_(self.nrFiles,"files processed") 238 print_("Reduced total size from",humanReadableSize(self.prevSize), 239 "to",humanReadableSize(self.nowSize), 240 ". This is",(100.*self.nowSize)/self.prevSize,"percent of the original") 241 if self.nrSkipped>0: 242 print_("Skipped",self.nrSkipped,"files because they were smaller than",self.bigSize) 243 if self.nrCompressed>0: 244 print_("Skipped",self.nrCompressed,"files because they are already compressed") 245 if self.nrProblems>0: 246 print_("Problems during the compression of",self.nrProblems,"files")
247 248 # Should work with Python3 and Python2 249