Package PyFoam :: Package Basics :: Module RunDatabase
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Basics.RunDatabase

  1  #  ICE Revision: $Id: $ 
  2  """ 
  3  Collects data about runs in a small SQLite database 
  4  """ 
  5   
  6  # don't look at it too closely. It's my first sqlite-code 
  7   
  8  import sqlite3 
  9  from os import path 
 10  import datetime 
 11  import re 
 12  import sys 
 13   
 14  from PyFoam.Error import error 
 15  from .CSVCollection import CSVCollection 
 16   
 17  from PyFoam.ThirdParty.six import print_,iteritems,integer_types 
 18  from PyFoam.ThirdParty.six import u as uniCode 
 19   
20 -class RunDatabase(object):
21 """ 22 Database with information about runs. To be queried etc 23 """ 24 25 separator="//" 26
27 - def __init__(self, 28 name, 29 create=False, 30 verbose=False):
31 """@param name: name of the file 32 @param create: should the database be created if it does not exist""" 33 34 self.verbose=verbose 35 if not path.exists(name): 36 if create==False: 37 error("Database",name,"does not exist") 38 else: 39 self.initDatabase(name) 40 41 self.db=sqlite3.connect(name) 42 self.db.row_factory=sqlite3.Row
43
44 - def initDatabase(self,name):
45 """Create a new database file""" 46 db=sqlite3.connect(name) 47 with db: 48 db.row_factory=sqlite3.Row 49 cursor=db.cursor() 50 cursor.execute("CREATE TABLE theRuns(runId INTEGER PRIMARY KEY, "+ 51 self.__normalize("insertionTime")+" TIMESTAMP)")
52
53 - def add(self,data):
54 """Add a dictionary with data to the database""" 55 self.__adaptDatabase(data) 56 57 runData=dict([("insertionTime",datetime.datetime.now())]+ \ 58 [(k,v) for k,v in iteritems(data) if type(v)!=dict]) 59 runID=self.__addContent("theRuns",runData) 60 61 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict]) 62 for tn,content in iteritems(subtables): 63 self.__addContent(tn+"Data", 64 dict(list(self.__flattenDict(content).items())+ 65 [("runId",runID)])) 66 67 self.db.commit()
68
69 - def __normalize(self,s):
70 """Normalize a column-name so that the case-insensitve column-names of SQlite 71 are no problem""" 72 73 if s in ["runId","dataId"]: 74 return s 75 result="" 76 for c in s: 77 if c.isupper() or c=="_": 78 result+="_"+c.lower() 79 else: 80 result+=c 81 return result
82
83 - def __denormalize(self,s):
84 """Denormalize the column name that was normalized by _normalize""" 85 86 result="" 87 underFound=False 88 89 for c in s: 90 if underFound: 91 underFound=False 92 result+=c.upper() 93 elif c=="_": 94 underFound=True 95 else: 96 result+=c 97 98 if underFound: 99 error("String",s,"was not correctly encoded") 100 101 return result
102
103 - def __addContent(self,table,data):
104 cursor=self.db.cursor() 105 runData={} 106 for k,v in iteritems(data): 107 if k=="runId": 108 runData[k]=v 109 elif isinstance(v,integer_types+(float,)): 110 runData[k]=float(v) 111 else: 112 runData[k]=uniCode(str(v)) 113 cols=self.__getColumns(table)[1:] 114 addData=[] 115 for c in cols: 116 try: 117 addData.append(runData[c]) 118 except KeyError: 119 addData.append(None) 120 addData=tuple(addData) 121 cSQL = "insert into "+table+" ("+ \ 122 ",".join(['"'+self.__normalize(c)+'"' for c in cols])+ \ 123 ") values ("+",".join(["?"]*len(addData))+")" 124 if self.verbose: 125 print_("Execute SQL",cSQL,"with",addData) 126 try: 127 cursor.execute(cSQL, addData) 128 except Exception: 129 e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' 130 print_("SQL-Expression:",cSQL) 131 print_("AddData:",addData) 132 raise e 133 134 return cursor.lastrowid
135
136 - def __adaptDatabase(self,data):
137 """Make sure that all the required columns and tables are there""" 138 139 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"') 140 tables=[ x["name"] for x in c.fetchall() ] 141 142 indata=dict([(k,v) for k,v in iteritems(data) if type(v)!=dict]) 143 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict]) 144 145 self.__addColumnsToTable("theRuns",indata) 146 147 for tn,content in iteritems(subtables): 148 if tn+"Data" not in tables: 149 if self.verbose: 150 print_("Adding table",tn) 151 self.db.execute("CREATE TABLE "+tn+"Data (dataId INTEGER PRIMARY KEY, runId INTEGER)") 152 self.__addColumnsToTable(tn+"Data", 153 self.__flattenDict(content))
154
155 - def __flattenDict(self,oData,prefix=""):
156 data=[(prefix+k,v) for k,v in iteritems(oData) if type(v)!=dict] 157 subtables=dict([(k,v) for k,v in iteritems(oData) if type(v)==dict]) 158 for name,val in iteritems(subtables): 159 data+=list(self.__flattenDict(val,prefix+name+self.separator).items()) 160 if self.verbose: 161 print_("Flattened",oData,"to",data) 162 return dict(data)
163
164 - def __getColumns(self,tablename):
165 c=self.db.execute('SELECT * from '+tablename) 166 result=[] 167 for desc in c.description: 168 if desc[0] in ['dataId','runId']: 169 result.append(desc[0]) 170 else: 171 result.append(self.__denormalize(desc[0])) 172 173 return result
174
175 - def __addColumnsToTable(self,table,data):
176 columns=self.__getColumns(table) 177 178 for k,v in iteritems(data): 179 if k not in columns: 180 if self.verbose: 181 print_("Adding:",k,"to",table) 182 if isinstance(v,integer_types+(float,)): 183 self.db.execute('ALTER TABLE "%s" ADD COLUMN "%s" REAL' % 184 (table,self.__normalize(k))) 185 else: 186 self.db.execute('ALTER TABLE "%s" ADD COLUMN "%s" TEXT' % 187 (table,self.__normalize(k)))
188
189 - def dumpToCSV(self, 190 fname, 191 selection=None, 192 disableRunData=None, 193 pandasFormat=True, 194 excel=False):
195 """Dump the contents of the database to a csv-file 196 @param name: the CSV-file 197 @param selection: list of regular expressions. Only data 198 entries fitting those will be added to the CSV-file (except 199 for the basic run). If unset all data will be written""" 200 file=CSVCollection(fname) 201 202 runCursor=self.db.cursor() 203 runCursor.execute("SELECT * from theRuns") 204 205 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"') 206 tables=[ x["name"] for x in c.fetchall() ] 207 208 allData=set() 209 writtenData=set() 210 211 disabledStandard=set() 212 213 for d in runCursor: 214 id=d['runId'] 215 if self.verbose: 216 print_("Dumping run",id) 217 for k in list(d.keys()): 218 writeEntry=True 219 if disableRunData: 220 for e in disableRunData: 221 exp=re.compile(e) 222 if not exp.search(self.__denormalize(k)) is None: 223 writeEntry=False 224 break 225 if writeEntry: 226 file[k]=d[k] 227 else: 228 disabledStandard.add(k) 229 for t in tables: 230 if t=="theRuns": 231 namePrefix="runInfo" 232 else: 233 namePrefix=t[:-4] 234 dataCursor=self.db.cursor() 235 dataCursor.execute("SELECT * FROM "+t+" WHERE runId=?", 236 (str(id),)) 237 data=dataCursor.fetchall() 238 if len(data)>1: 239 error(len(data),"data items found for id ",id, 240 "in table",t,".Need exactly 1") 241 elif len(data)<1: 242 continue 243 for k in list(data[0].keys()): 244 if k in ["dataId","runId"]: 245 continue 246 if k in disabledStandard: 247 continue 248 name=namePrefix+self.separator+self.__denormalize(k) 249 allData.add(name) 250 writeEntry=True 251 if selection: 252 writeEntry=False 253 for e in selection: 254 exp=re.compile(e) 255 if exp.search(name): 256 writeEntry=True 257 break 258 if writeEntry: 259 writtenData.add(name) 260 file[name]=data[0][k] 261 262 file.write() 263 264 if self.verbose: 265 sep="\n " 266 if allData==writtenData: 267 print_("Added all data entries:",sep,sep.join(sorted(allData)),sep="") 268 else: 269 print_("Added parameters:",sep,sep.join(sorted(writtenData)), 270 "\nUnwritten data:",sep,sep.join(sorted(allData-writtenData)),sep="") 271 if len(disabledStandard)>0: 272 print_("Disabled standard entries:",sep,sep.join(sorted(disabledStandard)),sep="") 273 274 f=file(pandasFormat) 275 if excel: 276 file(True).to_excel(fname) 277 278 if not f is None: 279 return f 280 else: 281 # retry by forcing to numpy 282 return file(False)
283 284 # Should work with Python3 and Python2 285