1
2 """
3 Collects data about runs in a small SQLite database
4 """
5
6
7
8 import sqlite3
9 from os import path
10 import datetime
11 import re
12 import sys
13
14 from PyFoam.Error import error
15 from .CSVCollection import CSVCollection
16
17 from PyFoam.ThirdParty.six import print_,iteritems,integer_types
18 from PyFoam.ThirdParty.six import u as uniCode
19
21 """
22 Database with information about runs. To be queried etc
23 """
24
25 separator="//"
26
27 - def __init__(self,
28 name,
29 create=False,
30 verbose=False):
31 """@param name: name of the file
32 @param create: should the database be created if it does not exist"""
33
34 self.verbose=verbose
35 if not path.exists(name):
36 if create==False:
37 error("Database",name,"does not exist")
38 else:
39 self.initDatabase(name)
40
41 self.db=sqlite3.connect(name)
42 self.db.row_factory=sqlite3.Row
43
45 """Create a new database file"""
46 db=sqlite3.connect(name)
47 with db:
48 db.row_factory=sqlite3.Row
49 cursor=db.cursor()
50 cursor.execute("CREATE TABLE theRuns(runId INTEGER PRIMARY KEY, "+
51 self.__normalize("insertionTime")+" TIMESTAMP)")
52
54 """Add a dictionary with data to the database"""
55 self.__adaptDatabase(data)
56
57 runData=dict([("insertionTime",datetime.datetime.now())]+ \
58 [(k,v) for k,v in iteritems(data) if type(v)!=dict])
59 runID=self.__addContent("theRuns",runData)
60
61 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict])
62 for tn,content in iteritems(subtables):
63 self.__addContent(tn+"Data",
64 dict(list(self.__flattenDict(content).items())+
65 [("runId",runID)]))
66
67 self.db.commit()
68
70 """Normalize a column-name so that the case-insensitve column-names of SQlite
71 are no problem"""
72
73 if s in ["runId","dataId"]:
74 return s
75 result=""
76 for c in s:
77 if c.isupper() or c=="_":
78 result+="_"+c.lower()
79 else:
80 result+=c
81 return result
82
84 """Denormalize the column name that was normalized by _normalize"""
85
86 result=""
87 underFound=False
88
89 for c in s:
90 if underFound:
91 underFound=False
92 result+=c.upper()
93 elif c=="_":
94 underFound=True
95 else:
96 result+=c
97
98 if underFound:
99 error("String",s,"was not correctly encoded")
100
101 return result
102
103 - def __addContent(self,table,data):
104 cursor=self.db.cursor()
105 runData={}
106 for k,v in iteritems(data):
107 if k=="runId":
108 runData[k]=v
109 elif isinstance(v,integer_types+(float,)):
110 runData[k]=float(v)
111 else:
112 runData[k]=uniCode(str(v))
113 cols=self.__getColumns(table)[1:]
114 addData=[]
115 for c in cols:
116 try:
117 addData.append(runData[c])
118 except KeyError:
119 addData.append(None)
120 addData=tuple(addData)
121 cSQL = "insert into "+table+" ("+ \
122 ",".join(['"'+self.__normalize(c)+'"' for c in cols])+ \
123 ") values ("+",".join(["?"]*len(addData))+")"
124 if self.verbose:
125 print_("Execute SQL",cSQL,"with",addData)
126 try:
127 cursor.execute(cSQL, addData)
128 except Exception:
129 e = sys.exc_info()[1]
130 print_("SQL-Expression:",cSQL)
131 print_("AddData:",addData)
132 raise e
133
134 return cursor.lastrowid
135
137 """Make sure that all the required columns and tables are there"""
138
139 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"')
140 tables=[ x["name"] for x in c.fetchall() ]
141
142 indata=dict([(k,v) for k,v in iteritems(data) if type(v)!=dict])
143 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict])
144
145 self.__addColumnsToTable("theRuns",indata)
146
147 for tn,content in iteritems(subtables):
148 if tn+"Data" not in tables:
149 if self.verbose:
150 print_("Adding table",tn)
151 self.db.execute("CREATE TABLE "+tn+"Data (dataId INTEGER PRIMARY KEY, runId INTEGER)")
152 self.__addColumnsToTable(tn+"Data",
153 self.__flattenDict(content))
154
156 data=[(prefix+k,v) for k,v in iteritems(oData) if type(v)!=dict]
157 subtables=dict([(k,v) for k,v in iteritems(oData) if type(v)==dict])
158 for name,val in iteritems(subtables):
159 data+=list(self.__flattenDict(val,prefix+name+self.separator).items())
160 if self.verbose:
161 print_("Flattened",oData,"to",data)
162 return dict(data)
163
165 c=self.db.execute('SELECT * from '+tablename)
166 result=[]
167 for desc in c.description:
168 if desc[0] in ['dataId','runId']:
169 result.append(desc[0])
170 else:
171 result.append(self.__denormalize(desc[0]))
172
173 return result
174
188
189 - def dumpToCSV(self,
190 fname,
191 selection=None,
192 disableRunData=None,
193 pandasFormat=True,
194 excel=False):
195 """Dump the contents of the database to a csv-file
196 @param name: the CSV-file
197 @param selection: list of regular expressions. Only data
198 entries fitting those will be added to the CSV-file (except
199 for the basic run). If unset all data will be written"""
200 file=CSVCollection(fname)
201
202 runCursor=self.db.cursor()
203 runCursor.execute("SELECT * from theRuns")
204
205 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"')
206 tables=[ x["name"] for x in c.fetchall() ]
207
208 allData=set()
209 writtenData=set()
210
211 disabledStandard=set()
212
213 for d in runCursor:
214 id=d['runId']
215 if self.verbose:
216 print_("Dumping run",id)
217 for k in list(d.keys()):
218 writeEntry=True
219 if disableRunData:
220 for e in disableRunData:
221 exp=re.compile(e)
222 if not exp.search(self.__denormalize(k)) is None:
223 writeEntry=False
224 break
225 if writeEntry:
226 file[k]=d[k]
227 else:
228 disabledStandard.add(k)
229 for t in tables:
230 if t=="theRuns":
231 namePrefix="runInfo"
232 else:
233 namePrefix=t[:-4]
234 dataCursor=self.db.cursor()
235 dataCursor.execute("SELECT * FROM "+t+" WHERE runId=?",
236 (str(id),))
237 data=dataCursor.fetchall()
238 if len(data)>1:
239 error(len(data),"data items found for id ",id,
240 "in table",t,".Need exactly 1")
241 elif len(data)<1:
242 continue
243 for k in list(data[0].keys()):
244 if k in ["dataId","runId"]:
245 continue
246 if k in disabledStandard:
247 continue
248 name=namePrefix+self.separator+self.__denormalize(k)
249 allData.add(name)
250 writeEntry=True
251 if selection:
252 writeEntry=False
253 for e in selection:
254 exp=re.compile(e)
255 if exp.search(name):
256 writeEntry=True
257 break
258 if writeEntry:
259 writtenData.add(name)
260 file[name]=data[0][k]
261
262 file.write()
263
264 if self.verbose:
265 sep="\n "
266 if allData==writtenData:
267 print_("Added all data entries:",sep,sep.join(sorted(allData)),sep="")
268 else:
269 print_("Added parameters:",sep,sep.join(sorted(writtenData)),
270 "\nUnwritten data:",sep,sep.join(sorted(allData-writtenData)),sep="")
271 if len(disabledStandard)>0:
272 print_("Disabled standard entries:",sep,sep.join(sorted(disabledStandard)),sep="")
273
274 f=file(pandasFormat)
275 if excel:
276 file(True).to_excel(fname)
277
278 if not f is None:
279 return f
280 else:
281
282 return file(False)
283
284
285