julian.kunkel/ddn-ime-evaluation
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

DB creator

Browse Source
This commit is contained in:
Eugen Betke 2018-10-22 19:07:54 +02:00
parent fe62d59925
commit 75ad720055
1 changed files with 46 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
mkdb.py
View File

@ -19,19 +19,20 @@ def parse(filename, conn):
exptype = 0
data = {}
metadata = {}
with open(filename, "r") as f:
data["filename"] = filename
metadata["filename"] = filename
for line in f:
#COUNT:1#NN:1#PPN:4#API:POSIX#T:10485760.txt
m = re.match("COUNT:([0-9]+)#NN:([0-9]+)#PPN:([0-9]+)#API:([\w]+)#T:([0-9]+).txt", os.path.basename(filename))
if (m):
data["count"] = int(m.group(1))
data["nn"] = int(m.group(2))
data["ppn"] = int(m.group(3))
data["api"] = m.group(4)
data["tsize"] = m.group(5)
metadata["count"] = int(m.group(1))
metadata["nn"] = int(m.group(2))
metadata["ppn"] = int(m.group(3))
metadata["api"] = m.group(4)
metadata["tsize"] = m.group(5)
else:
print('couldn\'t parse', os.path.basename(filename))
@ -41,40 +42,49 @@ def parse(filename, conn):
m = re.match("Command line used: .* -s[\s]+([0-9.]+)[\s]+-t[\s]+([0-9.]+)[\s]+-b[\s]+([0-9.]+)[\s]+-o.*", line)
if (m):
data["fsize"] = float(m.group(1)) * float(m.group(3)) * data["ppn"] * data["nn"]
#print(data["fsize"], float(m.group(1)), float(m.group(3)), data["nn"], data["ppn"])
#m = re.match("[\s]+aggregate filesize = ([0-9.]+)[\s]+.*", line)
metadata["fsize"] = float(m.group(1)) * float(m.group(3)) * data["ppn"] * data["nn"]
m = re.match("[\s]+aggregate filesize = (.*)", line)
if (m):
data["fsize_ctl"] = m.group(1)
metadata["fsize_ctl"] = m.group(1)
m = re.match("read[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]*$", line)
if (m):
data["read"] = float(m.group(1))
data["ropen"] = float(m.group(4))
data["rio"] = float(m.group(5))
data["rclose"] = float(m.group(6))
data["rtotal"] = float(m.group(7))
if m.group(8) not in data:
data[m.group(8)] = dict()
data[m.group(8)]["read"] = float(m.group(1))
data[m.group(8)]["ropen"] = float(m.group(4))
data[m.group(8)]["rio"] = float(m.group(5))
data[m.group(8)]["rclose"] = float(m.group(6))
data[m.group(8)]["rtotal"] = float(m.group(7))
data[m.group(8)]["riter"] = float(m.group(8))
data[m.group(8)].update(metadata)
m = re.match("write[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]+([0-9.]+)[\s]*$", line)
if (m):
data["write"] = float(m.group(1))
data["wopen"] = float(m.group(4))
data["wio"] = float(m.group(5))
data["wclose"] = float(m.group(6))
data["wtotal"] = float(m.group(7))
if m.group(8) not in data:
data[m.group(8)] = dict()
data[m.group(8)] = {}
data[m.group(8)]["write"] = float(m.group(1))
data[m.group(8)]["wopen"] = float(m.group(4))
data[m.group(8)]["wio"] = float(m.group(5))
data[m.group(8)]["wclose"] = float(m.group(6))
data[m.group(8)]["wtotal"] = float(m.group(7))
data[m.group(8)]["witer"] = float(m.group(8))
data[m.group(8)].update(metadata)
if len(data) == 28:
for iteration,entry in data.items():
if len(entry) == 19:
print("Success")
columns = ", ".join(data.keys())
placeholders = ':' + ', :'.join(data.keys())
columns = ", ".join(entry.keys())
placeholders = ':' + ', :'.join(entry.keys())
try:
conn.execute("INSERT INTO p (%s) VALUES (%s)" %(columns, placeholders), data)
conn.execute("INSERT INTO p (%s) VALUES (%s)" %(columns, placeholders), entry)
except sqlite3.IntegrityError as e:
print("Already imported")
else:
print("Error in file %s with tuples %s size %d"% (filename, data, len(data)))
print("Error in file %s with tuples %s size %d"% (filename, entry, len(entry)))
exptype += 1;
@ -95,17 +105,20 @@ try:
api text, \
tsize float, \
fsize float, \
fsize_ctl txt, \
ropen float, \
rio float, \
rclose float, \
rtotal float, \
read float, \
riter float, \
wopen float, \
wio float, \
wclose float, \
wtotal float, \
write float, \
primary key(filename) \
witer float, \
primary key(filename, witer, riter) \
)'
conn.execute(tbl)
except: