1
2 """
3 Collects data about runs in a small SQLite database
4 """
5
6
7
8 import sqlite3
9 from os import path
10 import datetime
11 import re
12 import sys
13
14 from PyFoam.Error import error
15 from .CSVCollection import CSVCollection
16
17 from PyFoam.ThirdParty.six import print_,iteritems,integer_types
18 from PyFoam.ThirdParty.six import u as uniCode
19
21 """
22 Database with information about runs. To be queried etc
23 """
24
25 separator="//"
26
27 - def __init__(self,
28 name,
29 create=False,
30 verbose=False):
31 """@param name: name of the file
32 @param create: should the database be created if it does not exist"""
33
34 self.verbose=verbose
35 if not path.exists(name):
36 if create==False:
37 error("Database",name,"does not exist")
38 else:
39 self.initDatabase(name)
40
41 self.db=sqlite3.connect(name)
42 self.db.row_factory=sqlite3.Row
43
45 """Create a new database file"""
46 db=sqlite3.connect(name)
47 with db:
48 db.row_factory=sqlite3.Row
49 cursor=db.cursor()
50 cursor.execute("CREATE TABLE theRuns(runId INTEGER PRIMARY KEY, "+
51 "insertionTime TIMESTAMP)")
52
54 """Add a dictionary with data to the database"""
55 self.__adaptDatabase(data)
56
57 runData=dict([("insertionTime",datetime.datetime.now())]+ \
58 [(k,v) for k,v in iteritems(data) if type(v)!=dict])
59 runID=self.__addContent("theRuns",runData)
60
61 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict])
62 for tn,content in iteritems(subtables):
63 self.__addContent(tn+"Data",
64 dict(list(self.__flattenDict(content).items())+
65 [("runId",runID)]))
66
67 self.db.commit()
68
69 - def __addContent(self,table,data):
70 cursor=self.db.cursor()
71 runData={}
72 for k,v in iteritems(data):
73 if k=="runId":
74 runData[k]=v
75 elif isinstance(v,integer_types+(float,)):
76 runData[k]=float(v)
77 else:
78 runData[k]=uniCode(str(v))
79 cols=self.__getColumns(table)[1:]
80 addData=[]
81 for c in cols:
82 try:
83 addData.append(runData[c])
84 except KeyError:
85 addData.append(None)
86 addData=tuple(addData)
87 cSQL = "insert into "+table+" ("+ \
88 ",".join(['"'+c+'"' for c in cols])+ \
89 ") values ("+",".join(["?"]*len(addData))+")"
90 if self.verbose:
91 print_("Execute SQL",cSQL,"with",addData)
92 try:
93 cursor.execute(cSQL, addData)
94 except Exception:
95 e = sys.exc_info()[1]
96 print_("SQL-Expression:",cSQL)
97 print_("AddData:",addData)
98 raise e
99
100 return cursor.lastrowid
101
103 """Make sure that all the required columns and tables are there"""
104
105 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"')
106 tables=[ x["name"] for x in c.fetchall() ]
107
108 indata=dict([(k,v) for k,v in iteritems(data) if type(v)!=dict])
109 subtables=dict([(k,v) for k,v in iteritems(data) if type(v)==dict])
110
111 self.__addColumnsToTable("theRuns",indata)
112
113 for tn,content in iteritems(subtables):
114 if tn+"Data" not in tables:
115 if self.verbose:
116 print_("Adding table",tn)
117 self.db.execute("CREATE TABLE "+tn+"Data (dataId INTEGER PRIMARY KEY, runId INTEGER)")
118 self.__addColumnsToTable(tn+"Data",
119 self.__flattenDict(content))
120
122 data=[(prefix+k,v) for k,v in iteritems(oData) if type(v)!=dict]
123 subtables=dict([(k,v) for k,v in iteritems(oData) if type(v)==dict])
124 for name,val in iteritems(subtables):
125 data+=list(self.__flattenDict(val,prefix+name+self.separator).items())
126 if self.verbose:
127 print_("Flattened",oData,"to",data)
128 return dict(data)
129
131 c=self.db.execute('SELECT * from '+tablename)
132 return [desc[0] for desc in c.description]
133
147
149 """Dump the contents of the database to a csv-file
150 @param name: the CSV-file
151 @param selection: list of regular expressions. Only data
152 entries fitting those will be added to the CSV-file (except
153 for the basic run). If unset all data will be written"""
154 file=CSVCollection(name)
155
156 runCursor=self.db.cursor()
157 runCursor.execute("SELECT * from theRuns")
158
159 c=self.db.execute('SELECT name FROM sqlite_master WHERE type = "table"')
160 tables=[ x["name"] for x in c.fetchall() ]
161
162 allData=set()
163 writtenData=set()
164
165 for d in runCursor:
166 id=d['runId']
167 if self.verbose:
168 print_("Dumping run",id)
169 for k in list(d.keys()):
170 file[k]=d[k]
171 for t in tables:
172 if t=="theRuns":
173 continue
174 namePrefix=t[:-4]
175 dataCursor=self.db.cursor()
176 dataCursor.execute("SELECT * FROM "+t+" WHERE runId=?",
177 (str(id),))
178 data=dataCursor.fetchall()
179 if len(data)>1:
180 error(len(data),"data items found for id ",id,
181 "in table",t,".Need exactly 1")
182 elif len(data)<1:
183 continue
184 for k in list(data[0].keys()):
185 if k in ["dataId","runId"]:
186 continue
187 name=namePrefix+" "+k
188 allData.add(name)
189 writeEntry=True
190 if selection:
191 writeEntry=False
192 for e in selection:
193 exp=re.compile(e)
194 if exp.search(name):
195 writeEntry=True
196 break
197 if writeEntry:
198 writtenData.add(name)
199 file[name]=data[0][k]
200
201 file.write()
202 if self.verbose:
203 if allData==writtenData:
204 print_("Added all data entries",allData)
205 else:
206 print_("Added parameters",writtenData,"of possible",allData)
207
208
209