julian.kunkel
/
mistral-io-datasets
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

nai

This commit is contained in:
eugen.betke 2020-10-13 12:52:30 +02:00
parent 92272913e3
commit e9e9143250
2 changed files with 66 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/kstest/run.sh
View File

@ -15,8 +15,8 @@ jobids=( ${jobids[@]} 5024292 )
set -x
for jobid in ${jobids[@]}; do
sim_fn="$output_dir/ks_similarities_$jobid.csv"
progress_fn="$output_dir/ks_progress_$jobid.csv"
log_fn="$output_dir/ks_fail_$jobid.log"
sim_fn="$output_dir/ks2_similarities_$jobid.csv"
progress_fn="$output_dir/ks2_progress_$jobid.csv"
log_fn="$output_dir/ks2_fail_$jobid.log"
time cargo run --release -- $dataset_fn $jobid $sim_fn $progress_fn $log_fn
done

45
tools/kstest/src/main.rs
View File

@ -24,15 +24,15 @@ pub type QCodings = HashMap<Jobid, JobCoding>;
#[derive(Debug, Deserialize)]
pub struct Record {
jobid: u32,
ks_md_file_create: String,
ks_md_file_delete: String,
ks_md_mod: String,
ks_md_other: String,
ks_md_read: String,
ks_read_bytes: String,
ks_read_calls: String,
ks_write_bytes: String,
ks_write_calls: String,
md_file_create: String,
md_file_delete: String,
md_mod: String,
md_other: String,
md_read: String,
read_bytes: String,
read_calls: String,
write_bytes: String,
write_calls: String,
}
#[derive(Debug, Serialize)]
@ -76,15 +76,15 @@ fn run(dataset_fn: String, jobid: Jobid, similarities_fn: String, progress_fn: S
for result in rdr.deserialize() {
let record: Record = result.expect("bla bla");
let q_coding = vec![
convert_to_coding(record.ks_md_file_create),
convert_to_coding(record.ks_md_file_delete),
convert_to_coding(record.ks_md_mod),
convert_to_coding(record.ks_md_other),
convert_to_coding(record.ks_md_read),
convert_to_coding(record.ks_read_bytes),
convert_to_coding(record.ks_read_calls),
convert_to_coding(record.ks_write_bytes),
convert_to_coding(record.ks_write_calls),
convert_to_coding(record.md_file_create),
convert_to_coding(record.md_file_delete),
convert_to_coding(record.md_mod),
convert_to_coding(record.md_other),
convert_to_coding(record.md_read),
convert_to_coding(record.read_bytes),
convert_to_coding(record.read_calls),
convert_to_coding(record.write_bytes),
convert_to_coding(record.write_calls),
];
// Filter Zero-Jobs
@ -102,13 +102,15 @@ fn run(dataset_fn: String, jobid: Jobid, similarities_fn: String, progress_fn: S
let mut wtr_progress = csv::Writer::from_writer(&progress_file);
let mut counter = 1;
let mut avail_codings: Vec<(u32, &JobCoding)>;
avail_codings = q_codings.iter().map(|(k, v)| (*k, v)).collect();
let mut similarities: Vec<(Jobid, Similarity, bool)> = Vec::new();
let log_file = File::create(&log_fn).expect("Unable to open");
let mut log_file = LineWriter::new(log_file);
for i in 0..10 {
let mut avail_codings: Vec<(u32, &JobCoding)>;
avail_codings = q_codings.iter().map(|(k, v)| (*k, v)).collect();
let probe = q_codings[&jobid].clone();
let mut start_chunk = chrono::Utc::now();
let start = start_chunk;
@ -116,7 +118,7 @@ fn run(dataset_fn: String, jobid: Jobid, similarities_fn: String, progress_fn: S
if (counter % 10_000) == 0 {
let stop_chunk = chrono::Utc::now();
let progress_row = ProgressRow {
iteration: 0,
iteration: i,
alg_id: alg_id,
alg_name: String::from(alg_name),
jobs_done: counter,
@ -151,6 +153,7 @@ fn run(dataset_fn: String, jobid: Jobid, similarities_fn: String, progress_fn: S
similarities.push((jobid, similarity, err));
counter += 1;
}
}
let mut status_map: HashMap<bool, String> = HashMap::new();