From e3090c5c23f3360aafc4292e11a1523d4203707b Mon Sep 17 00:00:00 2001 From: Eugen Betke Date: Tue, 23 Oct 2018 10:25:28 +0200 Subject: [PATCH] Analysis and makeDB script fixed --- eval_analysis.R | 68 +++++++++++++++++-------------------------------- mkdb.py | 7 ++++- 2 files changed, 30 insertions(+), 45 deletions(-) diff --git a/eval_analysis.R b/eval_analysis.R index 228812a2f..afe7416dc 100755 --- a/eval_analysis.R +++ b/eval_analysis.R @@ -30,8 +30,8 @@ connection = dbConnect(SQLite(), dbname=file_db) #dbdata = dbGetQuery(connection,'select * from p where tag=="mpio-individual"') #dbdata = dbGetQuery(connection,'select *, (x*y*z) as blocksize from p where count=8') #dbdata = dbGetQuery(connection,'select * from p where count<5') -dbdata = dbGetQuery(connection,'select * from p where ppn==1 or ppn=4 or ppn=8') -dbdata[,"blocksize"] = dbdata$x * dbdata$y * dbdata$z * 4 +dbdata = dbGetQuery(connection,'select * from p') +dbdata[,"blocksize"] = dbdata$tsize summary(dbdata) @@ -57,30 +57,26 @@ dims_list = data.frame(h, w, event) # df is a data frame fss = unique(dbdata$fs) for (fs in fss) { data1 = dbdata[fs == dbdata$fs, ] -ifaces = unique(data1$iface) +apis = unique(data1$api) -for (iface in ifaces) { -data2 = data1[iface == data1$iface, ] +print(fs) + +for (api in apis) { +data2 = data1[api == data1$api, ] apps = unique(data2$app) +print(api) + for (app in apps) { data3 = data2[app == data2$app, ] types = unique(data3$type) +print(app) + for (type in types) { -data4 = data3[type == data3$type, ] -chunkeds = unique(data4$chunked) +data = data3[type == data3$type, ] -for (chunked in chunkeds) { -data5 = data4[chunked == data4$chunked, ] -filleds = unique(data4$filled) - -for (filled in filleds) { -data6 = data5[filled == data5$filled, ] -unlimiteds = unique(data5$unlimited) - -for (unlimited in unlimiteds) { -data = data6[unlimited == data5$unlimited, ] +print(type) ggplot(data=data, aes(x=nn, y=write, colour=as.factor(blocksize/1024), group=blocksize), ymin=0) + #ggtitle("Write") + @@ -93,13 +89,13 @@ data = data6[unlimited == data5$unlimited, ] scale_x_continuous(breaks = c(unique(data$nn))) + scale_color_manual(name="Blocksize in KiB: ", values=c('#999999','#E69F00', '#56B4E9', '#000000'), breaks=sort(unique(data$blocksize)/1024)) + #stat_summary(fun.y="median", geom="line", aes(group=factor(blocksize))) + - stat_summary(fun.y="max", geom="line", aes(group=factor(blocksize))) + + stat_summary(fun.y="mean", geom="line", aes(group=factor(blocksize))) + #geom_boxplot() geom_point() - filename_eps = sprintf("%s/performance_%s_%s_%s_%s_CHUNK:%s_FILL:%s_LIM:%s_%s.eps", folder_out, app, fs, iface, type, chunked, filled, unlimited, "write") - filename_png = sprintf("%s/performance_%s_%s_%s_%s_CHUNK:%s_FILL:%s_LIM:%s_%s.png", folder_out, app, fs, iface, type, chunked, filled, unlimited, "write") - ggsave(filename_png, width = 6, height = 4) - ggsave(filename_eps, width = 6, height = 4) + filename_eps = sprintf("%s/performance_%s_%s_%s_%s_%s.eps", folder_out, app, fs, api, type, "write") + filename_png = sprintf("%s/performance_%s_%s_%s_%s_%s.png", folder_out, app, fs, api, type, "write") + ggsave(filename_png, width = 6, height = 10) + ggsave(filename_eps, width = 6, height = 10) system(sprintf("epstopdf %s", filename_eps)) system(sprintf("rm %s", filename_eps)) @@ -114,30 +110,14 @@ data = data6[unlimited == data5$unlimited, ] scale_x_continuous(breaks = c(unique(data$nn))) + scale_color_manual(name="Blocksize in KiB: ", values=c('#999999','#E69F00', '#56B4E9', '#000000'), breaks=sort(unique(data$blocksize)/1024)) + #stat_summary(fun.y="median", geom="line", aes(group=factor(blocksize))) + - stat_summary(fun.y="max", geom="line", aes(group=factor(blocksize))) + + stat_summary(fun.y="mean", geom="line", aes(group=factor(blocksize))) + #geom_boxplot() geom_point() - filename_eps = sprintf("%s/performance_%s_%s_%s_%s_CHUNK:%s_FILL:%s_LIM:%s_%s.eps", folder_out, app, fs, iface, type, chunked, filled, unlimited, "read") - filename_png = sprintf("%s/performance_%s_%s_%s_%s_CHUNK:%s_FILL:%s_LIM:%s_%s.png", folder_out, app, fs, iface, type, chunked, filled, unlimited, "read") - ggsave(filename_png, width = 6, height = 4) - ggsave(filename_eps, width = 6, height = 4) + filename_eps = sprintf("%s/performance_%s_%s_%s_%s_%s.eps", folder_out, app, fs, api, type, "read") + filename_png = sprintf("%s/performance_%s_%s_%s_%s_%s.png", folder_out, app, fs, api, type, "read") + ggsave(filename_png, width = 6, height = 10) + ggsave(filename_eps, width = 6, height = 10) system(sprintf("epstopdf %s", filename_eps)) system(sprintf("rm %s", filename_eps)) - - #ggplot(data=data, aes(x=blocksize, y=read, colour=app, group=blocksize)) + - # ggtitle("Read") + - # facet_grid(ppn ~ nn, labeller = labeller(nn = as_labeller(nn_lab), ppn = as_labeller(ppn_lab))) + - # xlab("Blocksize in KiB") + - # ylab("Performance in MiB/s") + - # theme(axis.text.x=element_text(angle=90, hjust=0.95, vjust=0.5)) + - # scale_y_log10() + - # scale_x_log10(breaks = breaks, labels=breaks/1024) + - # geom_boxplot() - ##geom_line() + - ##geom_point() - #filename_eps = sprintf("%s/performance_%s_%s_%s_%s_%s.eps", folder_out, app, fs, iface, type, "read") - #ggsave(filename_eps, width = 8, height = 6) - ##system(sprintf("epstopdf %s", filename_eps)) - -}}}}}}} +}}}} diff --git a/mkdb.py b/mkdb.py index bbb5e5f4a..4b07d3420 100755 --- a/mkdb.py +++ b/mkdb.py @@ -33,6 +33,9 @@ def parse(filename, conn): metadata["ppn"] = int(m.group(3)) metadata["api"] = m.group(4) metadata["tsize"] = m.group(5) + metadata["fs"] = "lustre" + metadata["app"] = "ior-default" + metadata["type"] = "random" else: print('couldn\'t parse', os.path.basename(filename)) @@ -75,7 +78,7 @@ def parse(filename, conn): for iteration,entry in data.items(): - if len(entry) == 19: + if len(entry) == 22: print("Success") columns = ", ".join(entry.keys()) placeholders = ':' + ', :'.join(entry.keys()) @@ -103,6 +106,8 @@ try: nn int, \ ppn int, \ api text, \ + fs text, \ + type text, \ tsize float, \ fsize float, \ fsize_ctl txt, \