mistral-io-datasets/datasets/compress.sh

#!/bin/bash

filenames=( $(ls job_codings_v*.csv) )
filenames=( ${filenames[@]} "clustering_progress.csv" )
filenames=( ${filenames[@]} $(ls job_metadata*.csv) )
filenames=( ${filenames[@]} $( ls job_similarities_*.csv ) )
filenames=( ${filenames[@]} $( ls sim_computation_times_*.csv ) )
filenames=( ${filenames[@]} $( ls ks_*.csv ) )

echo "${filenames[*]}"

for in_fn in ${filenames[@]}; do
    out_fn="${in_fn}.tar.xz"
    if [ -f "${in_fn}" ]; then
        if [ ! -f "${out_fn}" ]; then
            echo "Compressing ${in_fn}"
            tar -cf - "${in_fn}" | xz -9 > "${out_fn}"
        else
            echo "Skipping ${out_fn}. File exists."
        fi
    fi
done