Package PyFoam :: Package Basics :: Module RunDatabase
[hide private]
[frames] | no frames]

Source Code for Module PyFoam.Basics.RunDatabase

  1  #  ICE Revision: $Id: $ 
  2  """ 
  3  Collects data about runs in a small SQLite database 
  4  """ 
  5   
  6  # don't look at it too closely. It's my first sqlite-code 
  7   
  8  import sqlite3 
  9  from os import path 
 10  import datetime 
 11  import re 
 12  import sys 
 13   
 14  from PyFoam.Error import error 
 15  from .CSVCollection import CSVCollection 
 16   
 17  from PyFoam.ThirdParty.six import print_,iteritems,integer_types 
 18  from PyFoam.ThirdParty.six import u as uniCode 
 19   
20 -class RunDatabase(object):
21 """ 22 Database with information about runs. To be queried etc 23 """ 24 25 separator="//" 26
27 - def __init__(self, 28 name, 29 create=False, 30 verbose=False):
31 """@param name: name of the file 32 @param create: should the database be created if it does not exist""" 33 34 self.verbose=verbose 35 if not path.exists(name): 36 if create==False: 37 error("Database",name,"does not exist") 38 else: 39 self.initDatabase(name) 40 41 self.db=sqlite3.connect(name) 42 self.db.row_factory=sqlite3.Row
43
44 - def initDatabase(self,name):
45 """Create a new database file""" 46 db=sqlite3.connect(name) 47 with db: 48 db.row_factory=sqlite3.Row 49 cursor=db.cursor() 50 cursor.execute("CREATE TABLE theRuns(runId INTEGER PRIMARY KEY, "+ 51 "insertionTime TIMESTAMP)")
52
53 - def add(self,data):
54 """Add a dictionary with data to the database""" 55 self.__adaptDatabase(data) 56 57 runData=dict([("insertionTime",datetime.datetime.now())]+ \ 58 [(k,v) for k,v in iteritems(data) if type(v)!=dict]) 59 runID=self.__addContent("theRuns",runData) 60 61 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict]) 62 for tn,content in iteritems(subtables): 63 self.__addContent(tn+"Data", 64 dict(list(self.__flattenDict(content).items())+ 65 [("runId",runID)])) 66 67 self.db.commit()
68
69 - def __addContent(self,table,data):
70 cursor=self.db.cursor() 71 runData={} 72 for k,v in iteritems(data): 73 if k=="runId": 74 runData[k]=v 75 elif isinstance(v,integer_types+(float,)): 76 runData[k]=float(v) 77 else: 78 runData[k]=uniCode(str(v)) 79 cols=self.__getColumns(table)[1:] 80 addData=[] 81 for c in cols: 82 try: 83 addData.append(runData[c]) 84 except KeyError: 85 addData.append(None) 86 addData=tuple(addData) 87 cSQL = "insert into "+table+" ("+ \ 88 ",".join(['"'+c+'"' for c in cols])+ \ 89 ") values ("+",".join(["?"]*len(addData))+")" 90 if self.verbose: 91 print_("Execute SQL",cSQL,"with",addData) 92 try: 93 cursor.execute(cSQL, addData) 94 except Exception: 95 e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' 96 print_("SQL-Expression:",cSQL) 97 print_("AddData:",addData) 98 raise e 99 100 return cursor.lastrowid
101
102 - def __adaptDatabase(self,data):
103 """Make sure that all the required columns and tables are there""" 104 105 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"') 106 tables=[ x["name"] for x in c.fetchall() ] 107 108 indata=dict([(k,v) for k,v in iteritems(data) if type(v)!=dict]) 109 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict]) 110 111 self.__addColumnsToTable("theRuns",indata) 112 113 for tn,content in iteritems(subtables): 114 if tn+"Data" not in tables: 115 if self.verbose: 116 print_("Adding table",tn) 117 self.db.execute("CREATE TABLE "+tn+"Data (dataId INTEGER PRIMARY KEY, runId INTEGER)") 118 self.__addColumnsToTable(tn+"Data", 119 self.__flattenDict(content))
120
121 - def __flattenDict(self,oData,prefix=""):
122 data=[(prefix+k,v) for k,v in iteritems(oData) if type(v)!=dict] 123 subtables=dict([(k,v) for k,v in iteritems(oData) if type(v)==dict]) 124 for name,val in iteritems(subtables): 125 data+=list(self.__flattenDict(val,prefix+name+self.separator).items()) 126 if self.verbose: 127 print_("Flattened",oData,"to",data) 128 return dict(data)
129
130 - def __getColumns(self,tablename):
131 c=self.db.execute('SELECT * from '+tablename) 132 return [desc[0] for desc in c.description]
133
134 - def __addColumnsToTable(self,table,data):
135 columns=self.__getColumns(table) 136 137 for k,v in iteritems(data): 138 if k not in columns: 139 if self.verbose: 140 print_("Adding:",k,"to",table) 141 if isinstance(v,integer_types+(float,)): 142 self.db.execute('ALTER TABLE "%s" ADD COLUMN "%s" REAL' % 143 (table,k)) 144 else: 145 self.db.execute('ALTER TABLE "%s" ADD COLUMN "%s" TEXT' % 146 (table,k))
147
148 - def dumpToCSV(self,name,selection=None):
149 """Dump the contents of the database to a csv-file 150 @param name: the CSV-file 151 @param selection: list of regular expressions. Only data 152 entries fitting those will be added to the CSV-file (except 153 for the basic run). If unset all data will be written""" 154 file=CSVCollection(name) 155 156 runCursor=self.db.cursor() 157 runCursor.execute("SELECT * from theRuns") 158 159 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"') 160 tables=[ x["name"] for x in c.fetchall() ] 161 162 allData=set() 163 writtenData=set() 164 165 for d in runCursor: 166 id=d['runId'] 167 if self.verbose: 168 print_("Dumping run",id) 169 for k in list(d.keys()): 170 file[k]=d[k] 171 for t in tables: 172 if t=="theRuns": 173 continue 174 namePrefix=t[:-4] 175 dataCursor=self.db.cursor() 176 dataCursor.execute("SELECT * FROM "+t+" WHERE runId=?", 177 (str(id),)) 178 data=dataCursor.fetchall() 179 if len(data)>1: 180 error(len(data),"data items found for id ",id, 181 "in table",t,".Need exactly 1") 182 elif len(data)<1: 183 continue 184 for k in list(data[0].keys()): 185 if k in ["dataId","runId"]: 186 continue 187 name=namePrefix+" "+k 188 allData.add(name) 189 writeEntry=True 190 if selection: 191 writeEntry=False 192 for e in selection: 193 exp=re.compile(e) 194 if exp.search(name): 195 writeEntry=True 196 break 197 if writeEntry: 198 writtenData.add(name) 199 file[name]=data[0][k] 200 201 file.write() 202 if self.verbose: 203 if allData==writtenData: 204 print_("Added all data entries",allData) 205 else: 206 print_("Added parameters",writtenData,"of possible",allData)
207 208 # Should work with Python3 and Python2 209