Package PyFoam :: Package Basics :: Module RunDatabase
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Basics.RunDatabase

  1  #  ICE Revision: $Id: $ 
  2  """ 
  3  Collects data about runs in a small SQLite database 
  4  """ 
  5   
  6  # don't look at it too closely. It's my first sqlite-code 
  7   
  8  import sqlite3 
  9  from os import path 
 10  import datetime 
 11  import re 
 12  import sys 
 13   
 14  from PyFoam.Error import error 
 15  from .CSVCollection import CSVCollection 
 16   
 17  from PyFoam.ThirdParty.six import print_,iteritems,integer_types 
 18  from PyFoam.ThirdParty.six import u as uniCode 
 19   
20 -class RunDatabase(object):
21 """ 22 Database with information about runs. To be queried etc 23 """ 24 25 separator="//" 26
27 - def __init__(self, 28 name, 29 create=False, 30 verbose=False):
31 """@param name: name of the file 32 @param create: should the database be created if it does not exist""" 33 34 self.verbose=verbose 35 if not path.exists(name): 36 if create==False: 37 error("Database",name,"does not exist") 38 else: 39 self.initDatabase(name) 40 41 self.db=sqlite3.connect(name) 42 self.db.row_factory=sqlite3.Row
43
44 - def initDatabase(self,name):
45 """Create a new database file""" 46 db=sqlite3.connect(name) 47 with db: 48 db.row_factory=sqlite3.Row 49 cursor=db.cursor() 50 cursor.execute("CREATE TABLE theRuns(runId INTEGER PRIMARY KEY, "+ 51 "insertionTime TIMESTAMP)")
52
53 - def add(self,data):
54 """Add a dictionary with data to the database""" 55 self.__adaptDatabase(data) 56 57 runData=dict([("insertionTime",datetime.datetime.now())]+ \ 58 [(k,v) for k,v in iteritems(data) if type(v)!=dict]) 59 runID=self.__addContent("theRuns",runData) 60 61 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict]) 62 for tn,content in iteritems(subtables): 63 self.__addContent(tn+"Data", 64 dict(list(self.__flattenDict(content).items())+ 65 [("runId",runID)])) 66 67 self.db.commit()
68
69 - def __addContent(self,table,data):
70 cursor=self.db.cursor() 71 runData={} 72 for k,v in iteritems(data): 73 if k=="runId": 74 runData[k]=v 75 elif isinstance(v,integer_types+(float,)): 76 runData[k]=float(v) 77 else: 78 runData[k]=uniCode(str(v)) 79 cols=self.__getColumns(table)[1:] 80 addData=[] 81 for c in cols: 82 try: 83 addData.append(runData[c]) 84 except KeyError: 85 addData.append(None) 86 addData=tuple(addData) 87 cSQL = "insert into "+table+" ("+ \ 88 ",".join(['"'+c+'"' for c in cols])+ \ 89 ") values ("+",".join(["?"]*len(addData))+")" 90 if self.verbose: 91 print_("Execute SQL",cSQL,"with",addData) 92 try: 93 cursor.execute(cSQL, addData) 94 except Exception: 95 e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' 96 print_("SQL-Expression:",cSQL) 97 print_("AddData:",addData) 98 raise e 99 100 return cursor.lastrowid
101
102 - def __adaptDatabase(self,data):
103 """Make sure that all the required columns and tables are there""" 104 105 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"') 106 tables=[ x["name"] for x in c.fetchall() ] 107 108 indata=dict([(k,v) for k,v in iteritems(data) if type(v)!=dict]) 109 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict]) 110 111 self.__addColumnsToTable("theRuns",indata) 112 113 for tn,content in iteritems(subtables): 114 if tn+"Data" not in tables: 115 if self.verbose: 116 print_("Adding table",tn) 117 self.db.execute("CREATE TABLE "+tn+"Data (dataId INTEGER PRIMARY KEY, runId INTEGER)") 118 self.__addColumnsToTable(tn+"Data", 119 self.__flattenDict(content))
120
121 - def __flattenDict(self,oData,prefix=""):
122 data=[(prefix+k,v) for k,v in iteritems(oData) if type(v)!=dict] 123 subtables=dict([(k,v) for k,v in iteritems(oData) if type(v)==dict]) 124 for name,val in iteritems(subtables): 125 data+=list(self.__flattenDict(val,prefix+name+self.separator).items()) 126 if self.verbose: 127 print_("Flattened",oData,"to",data) 128 return dict(data)
129
130 - def __getColumns(self,tablename):
131 c=self.db.execute('SELECT * from '+tablename) 132 return [desc[0] for desc in c.description]
133
134 - def __addColumnsToTable(self,table,data):
135 columns=self.__getColumns(table) 136 137 for k,v in iteritems(data): 138 if k not in columns: 139 if self.verbose: 140 print_("Adding:",k,"to",table) 141 if isinstance(v,integer_types+(float,)): 142 self.db.execute('ALTER TABLE "%s" ADD COLUMN "%s" REAL' % 143 (table,k)) 144 else: 145 self.db.execute('ALTER TABLE "%s" ADD COLUMN "%s" TEXT' % 146 (table,k))
147
148 - def dumpToCSV(self, 149 fname, 150 selection=None, 151 disableRunData=None, 152 pandasFormat=True, 153 excel=False):
154 """Dump the contents of the database to a csv-file 155 @param name: the CSV-file 156 @param selection: list of regular expressions. Only data 157 entries fitting those will be added to the CSV-file (except 158 for the basic run). If unset all data will be written""" 159 file=CSVCollection(fname) 160 161 runCursor=self.db.cursor() 162 runCursor.execute("SELECT * from theRuns") 163 164 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"') 165 tables=[ x["name"] for x in c.fetchall() ] 166 167 allData=set() 168 writtenData=set() 169 170 disabledStandard=set() 171 172 for d in runCursor: 173 id=d['runId'] 174 if self.verbose: 175 print_("Dumping run",id) 176 for k in list(d.keys()): 177 writeEntry=True 178 if disableRunData: 179 for e in disableRunData: 180 exp=re.compile(e) 181 if exp.search(k): 182 writeEntry=False 183 break 184 if writeEntry: 185 file[k]=d[k] 186 else: 187 disabledStandard.add(k) 188 for t in tables: 189 if t=="theRuns": 190 continue 191 namePrefix=t[:-4] 192 dataCursor=self.db.cursor() 193 dataCursor.execute("SELECT * FROM "+t+" WHERE runId=?", 194 (str(id),)) 195 data=dataCursor.fetchall() 196 if len(data)>1: 197 error(len(data),"data items found for id ",id, 198 "in table",t,".Need exactly 1") 199 elif len(data)<1: 200 continue 201 for k in list(data[0].keys()): 202 if k in ["dataId","runId"]: 203 continue 204 name=namePrefix+self.separator+k 205 allData.add(name) 206 writeEntry=True 207 if selection: 208 writeEntry=False 209 for e in selection: 210 exp=re.compile(e) 211 if exp.search(name): 212 writeEntry=True 213 break 214 if writeEntry: 215 writtenData.add(name) 216 file[name]=data[0][k] 217 218 file.write() 219 220 if self.verbose: 221 sep="\n " 222 if allData==writtenData: 223 print_("Added all data entries:",sep,sep.join(allData),sep="") 224 else: 225 print_("Added parameters:",sep,sep.join(writtenData), 226 "\nUnwritten data:",sep,sep.join(allData-writtenData),sep="") 227 if len(disabledStandard)>0: 228 print_("Disabled standard entries:",sep,sep.join(disabledStandard),sep="") 229 230 f=file(pandasFormat) 231 if excel: 232 file(True).to_excel(fname) 233 234 if not f is None: 235 return f 236 else: 237 # retry by forcing to numpy 238 return file(False)
239 240 # Should work with Python3 and Python2 241