Commit fc943070 authored by Shane Snyder's avatar Shane Snyder
Browse files

final tweaks for darshan-job-summary graphs

parent ac263795
...@@ -78,10 +78,7 @@ my $cumul_meta_shared = 0; ...@@ -78,10 +78,7 @@ my $cumul_meta_shared = 0;
my $cumul_meta_indep = 0; my $cumul_meta_indep = 0;
my $first_data_line = 1; my $first_data_line = 1;
my $current_rank = 0;
my $current_hash = 0;
my %file_record_hash = (); my %file_record_hash = ();
my %fs_data = (); my %fs_data = ();
while($line = <PARSE_OUT>) while($line = <PARSE_OUT>)
...@@ -133,33 +130,23 @@ while($line = <PARSE_OUT>) ...@@ -133,33 +130,23 @@ while($line = <PARSE_OUT>)
@fields = split(/[\t ]+/, $line); @fields = split(/[\t ]+/, $line);
# encode the file system name to protect against special characters # encode the file system name to protect against special characters
$fields[5] = encode('latex', $fields[5]); $fields[6] = encode('latex', $fields[6]);
# is this our first piece of data? # is this our first piece of data?
if($first_data_line) if($first_data_line)
{ {
$current_rank = $fields[1];
$current_hash = $fields[2];
$first_data_line = 0; $first_data_line = 0;
} }
# is this a new file record? # is this a new file record?
if($fields[1] != $current_rank || $fields[2] != $current_hash) if (!defined $file_record_hash{$fields[2]})
{ {
# process previous record (if posix or mpiio record) $file_record_hash{$fields[2]} = ();
if ($fields[0] eq "POSIX" || $fields[0] eq "MPIIO") $file_record_hash{$fields[2]}{FILE_NAME} = $fields[5];
{ $file_record_hash{$fields[2]}{RANK} = $fields[1];
process_file_record($current_rank, $current_hash, \%file_record_hash);
}
# reset variables for next record
$current_rank = $fields[1];
$current_hash = $fields[2];
%file_record_hash = ();
$file_record_hash{FILE_NAME} = $fields[5];
} }
$file_record_hash{$fields[3]} = $fields[4]; $file_record_hash{$fields[2]}{$fields[3]} = $fields[4];
$summary{$fields[3]} += $fields[4]; $summary{$fields[3]} += $fields[4];
# accumulate independent and shared data as well as fs data # accumulate independent and shared data as well as fs data
...@@ -332,11 +319,9 @@ if ($first_data_line) ...@@ -332,11 +319,9 @@ if ($first_data_line)
exit(1); exit(1);
} }
# process last file record foreach $key (keys %file_record_hash)
$file_record_hash{FILE_NAME} = $fields[5];
if ($fields[0] eq "POSIX" || $fields[0] eq "MPIIO")
{ {
process_file_record($current_rank, $current_hash, \%file_record_hash); process_file_record($key, \%{$file_record_hash{$key}});
} }
# copy template files to tmp tmp_dir # copy template files to tmp tmp_dir
...@@ -477,6 +462,7 @@ close PATTERN; ...@@ -477,6 +462,7 @@ close PATTERN;
# table of common access sizes # table of common access sizes
open(ACCESS_TABLE, ">$tmp_dir/access-table.tex") || die("error opening output file:$!\n"); open(ACCESS_TABLE, ">$tmp_dir/access-table.tex") || die("error opening output file:$!\n");
print ACCESS_TABLE " print ACCESS_TABLE "
\\begin{threeparttable}
\\begin{tabular}{r|r|r} \\begin{tabular}{r|r|r}
\\multicolumn{3}{c}{ } \\\\ \\multicolumn{3}{c}{ } \\\\
\\multicolumn{3}{c}{Most Common Access Sizes} \\\\ \\multicolumn{3}{c}{Most Common Access Sizes} \\\\
...@@ -546,7 +532,7 @@ if ($tmp_access_count > 0) ...@@ -546,7 +532,7 @@ if ($tmp_access_count > 0)
if ($i == 0) { if ($i == 0) {
print ACCESS_TABLE " print ACCESS_TABLE "
\\hline \\hline
\\multirow{$tmp_access_count}{*}{MPI-IO \\textdagger} \& $value \& $mpiio_access_hash{$value} \\\\\n \\multirow{$tmp_access_count}{*}{MPI-IO \\textbf{\\ddag}} \& $value \& $mpiio_access_hash{$value} \\\\\n
"; ";
} }
else { else {
...@@ -562,197 +548,202 @@ print ACCESS_TABLE " ...@@ -562,197 +548,202 @@ print ACCESS_TABLE "
\\hline \\hline
\\end{tabular} \\end{tabular}
"; ";
if ($tmp_access_count > 0)
{
print ACCESS_TABLE "
\\begin{tablenotes}
\\item[\\normalsize \\textbf{\\ddag}] \\scriptsize NOTE: MPI-IO accesses are given in terms of aggregate datatype size.
\\end{tablenotes}
";
}
print ACCESS_TABLE "
\\end{threeparttable}
";
close ACCESS_TABLE; close ACCESS_TABLE;
# file count table # file count table
#open(TABLES, ">$tmp_dir/file-count-table.tex") || die("error opening output file:$!\n"); open(FILE_CNT_TABLE, ">$tmp_dir/file-count-table.tex") || die("error opening output file:$!\n");
#print TABLES " print FILE_CNT_TABLE "
#\\begin{tabular}{r|r|r|r} \\begin{tabular}{r|r|r|r}
#\\multicolumn{4}{c}{ } \\\\ \\multicolumn{4}{c}{ } \\\\
#\\multicolumn{4}{c}{File Count Summary} \\\\ \\multicolumn{4}{c}{File Count Summary} \\\\
#"; \\multicolumn{4}{c}{(estimated by I/O access offsets)} \\\\
#if($size_est_flag == 1) \\hline
#{ type \& number of files \& avg. size \& max size \\\\
#print TABLES " \\hline
#\\multicolumn{4}{c}{(estimated by I/O access offsets)} \\\\ \\hline
#"; ";
#}
#print TABLES " my $counter;
#\\hline my $sum;
#type \& number of files \& avg. size \& max size \\\\ my $max;
#\\hline my $key;
#\\hline my $avg;
#";
#my $counter; $counter = 0;
#my $sum; $sum = 0;
#my $max; $max = 0;
#my $key; foreach $key (keys %hash_files) {
#my $avg; $counter++;
# if($hash_files{$key}{'min_open_size'} >
#$counter = 0; $hash_files{$key}{'max_size'})
#$sum = 0; {
#$max = 0; $sum += $hash_files{$key}{'min_open_size'};
#foreach $key (keys %hash_files) { if($hash_files{$key}{'min_open_size'} > $max)
# $counter++; {
# if($hash_files{$key}{'min_open_size'} > $max = $hash_files{$key}{'min_open_size'};
# $hash_files{$key}{'max_size'}) }
# { }
# $sum += $hash_files{$key}{'min_open_size'}; else
# if($hash_files{$key}{'min_open_size'} > $max) {
# { $sum += $hash_files{$key}{'max_size'};
# $max = $hash_files{$key}{'min_open_size'}; if($hash_files{$key}{'max_size'} > $max)
# } {
# } $max = $hash_files{$key}{'max_size'};
# else }
# { }
# $sum += $hash_files{$key}{'max_size'}; }
# if($hash_files{$key}{'max_size'} > $max) if($counter > 0) { $avg = $sum / $counter; }
# { else { $avg = 0; }
# $max = $hash_files{$key}{'max_size'}; $avg = format_bytes($avg);
# } $max = format_bytes($max);
# } print FILE_CNT_TABLE "total opened \& $counter \& $avg \& $max \\\\\n";
#}
#if($counter > 0) { $avg = $sum / $counter; } $counter = 0;
#else { $avg = 0; } $sum = 0;
#$avg = format_bytes($avg); $max = 0;
#$max = format_bytes($max); foreach $key (keys %hash_files) {
#print TABLES "total opened \& $counter \& $avg \& $max \\\\\n"; if($hash_files{$key}{'was_read'} && !($hash_files{$key}{'was_written'}))
# {
#$counter = 0; $counter++;
#$sum = 0; if($hash_files{$key}{'min_open_size'} >
#$max = 0; $hash_files{$key}{'max_size'})
#foreach $key (keys %hash_files) { {
# if($hash_files{$key}{'was_read'} && !($hash_files{$key}{'was_written'})) $sum += $hash_files{$key}{'min_open_size'};
# { if($hash_files{$key}{'min_open_size'} > $max)
# $counter++; {
# if($hash_files{$key}{'min_open_size'} > $max = $hash_files{$key}{'min_open_size'};
# $hash_files{$key}{'max_size'}) }
# { }
# $sum += $hash_files{$key}{'min_open_size'}; else
# if($hash_files{$key}{'min_open_size'} > $max) {
# { $sum += $hash_files{$key}{'max_size'};
# $max = $hash_files{$key}{'min_open_size'}; if($hash_files{$key}{'max_size'} > $max)
# } {
# } $max = $hash_files{$key}{'max_size'};
# else }
# { }
# $sum += $hash_files{$key}{'max_size'}; }
# if($hash_files{$key}{'max_size'} > $max) }
# { if($counter > 0) { $avg = $sum / $counter; }
# $max = $hash_files{$key}{'max_size'}; else { $avg = 0; }
# } $avg = format_bytes($avg);
# } $max = format_bytes($max);
# } print FILE_CNT_TABLE "read-only files \& $counter \& $avg \& $max \\\\\n";
#}
#if($counter > 0) { $avg = $sum / $counter; } $counter = 0;
#else { $avg = 0; } $sum = 0;
#$avg = format_bytes($avg); $max = 0;
#$max = format_bytes($max); foreach $key (keys %hash_files) {
#print TABLES "read-only files \& $counter \& $avg \& $max \\\\\n"; if(!($hash_files{$key}{'was_read'}) && $hash_files{$key}{'was_written'})
# {
#$counter = 0; $counter++;
#$sum = 0; if($hash_files{$key}{'min_open_size'} >
#$max = 0; $hash_files{$key}{'max_size'})
#foreach $key (keys %hash_files) { {
# if(!($hash_files{$key}{'was_read'}) && $hash_files{$key}{'was_written'}) $sum += $hash_files{$key}{'min_open_size'};
# { if($hash_files{$key}{'min_open_size'} > $max)
# $counter++; {
# if($hash_files{$key}{'min_open_size'} > $max = $hash_files{$key}{'min_open_size'};
# $hash_files{$key}{'max_size'}) }
# { }
# $sum += $hash_files{$key}{'min_open_size'}; else
# if($hash_files{$key}{'min_open_size'} > $max) {
# { $sum += $hash_files{$key}{'max_size'};
# $max = $hash_files{$key}{'min_open_size'}; if($hash_files{$key}{'max_size'} > $max)
# } {
# } $max = $hash_files{$key}{'max_size'};
# else }
# { }
# $sum += $hash_files{$key}{'max_size'}; }
# if($hash_files{$key}{'max_size'} > $max) }
# { if($counter > 0) { $avg = $sum / $counter; }
# $max = $hash_files{$key}{'max_size'}; else { $avg = 0; }
# } $avg = format_bytes($avg);
# } $max = format_bytes($max);
# } print FILE_CNT_TABLE "write-only files \& $counter \& $avg \& $max \\\\\n";
#}
#if($counter > 0) { $avg = $sum / $counter; } $counter = 0;
#else { $avg = 0; } $sum = 0;
#$avg = format_bytes($avg); $max = 0;
#$max = format_bytes($max); foreach $key (keys %hash_files) {
#print TABLES "write-only files \& $counter \& $avg \& $max \\\\\n"; if($hash_files{$key}{'was_read'} && $hash_files{$key}{'was_written'})
# {
#$counter = 0; $counter++;
#$sum = 0; if($hash_files{$key}{'min_open_size'} >
#$max = 0; $hash_files{$key}{'max_size'})
#foreach $key (keys %hash_files) { {
# if($hash_files{$key}{'was_read'} && $hash_files{$key}{'was_written'}) $sum += $hash_files{$key}{'min_open_size'};
# { if($hash_files{$key}{'min_open_size'} > $max)
# $counter++; {
# if($hash_files{$key}{'min_open_size'} > $max = $hash_files{$key}{'min_open_size'};
# $hash_files{$key}{'max_size'}) }
# { }
# $sum += $hash_files{$key}{'min_open_size'}; else
# if($hash_files{$key}{'min_open_size'} > $max) {
# { $sum += $hash_files{$key}{'max_size'};
# $max = $hash_files{$key}{'min_open_size'}; if($hash_files{$key}{'max_size'} > $max)
# } {
# } $max = $hash_files{$key}{'max_size'};
# else }
# { }
# $sum += $hash_files{$key}{'max_size'}; }
# if($hash_files{$key}{'max_size'} > $max) }
# { if($counter > 0) { $avg = $sum / $counter; }
# $max = $hash_files{$key}{'max_size'}; else { $avg = 0; }
# } $avg = format_bytes($avg);
# } $max = format_bytes($max);
# } print FILE_CNT_TABLE "read/write files \& $counter \& $avg \& $max \\\\\n";
#}
#if($counter > 0) { $avg = $sum / $counter; } $counter = 0;
#else { $avg = 0; } $sum = 0;
#$avg = format_bytes($avg); $max = 0;
#$max = format_bytes($max); foreach $key (keys %hash_files) {
#print TABLES "read/write files \& $counter \& $avg \& $max \\\\\n"; if($hash_files{$key}{'was_written'} &&
# $hash_files{$key}{'min_open_size'} == 0 &&
#$counter = 0; $hash_files{$key}{'max_size'} > 0)
#$sum = 0; {
#$max = 0; $counter++;
#foreach $key (keys %hash_files) { if($hash_files{$key}{'min_open_size'} >
# if($hash_files{$key}{'was_written'} && $hash_files{$key}{'max_size'})
# $hash_files{$key}{'min_open_size'} == 0 && {
# $hash_files{$key}{'max_size'} > 0) $sum += $hash_files{$key}{'min_open_size'};
# { if($hash_files{$key}{'min_open_size'} > $max)
# $counter++; {
# if($hash_files{$key}{'min_open_size'} > $max = $hash_files{$key}{'min_open_size'};
# $hash_files{$key}{'max_size'}) }
# { }
# $sum += $hash_files{$key}{'min_open_size'}; else
# if($hash_files{$key}{'min_open_size'} > $max) {
# { $sum += $hash_files{$key}{'max_size'};
# $max = $hash_files{$key}{'min_open_size'}; if($hash_files{$key}{'max_size'} > $max)
# } {
# } $max = $hash_files{$key}{'max_size'};
# else }
# { }
# $sum += $hash_files{$key}{'max_size'}; }
# if($hash_files{$key}{'max_size'} > $max) }
# { if($counter > 0) { $avg = $sum / $counter; }
# $max = $hash_files{$key}{'max_size'}; else { $avg = 0; }
# } $avg = format_bytes($avg);
# } $max = format_bytes($max);
# } print FILE_CNT_TABLE "created files \& $counter \& $avg \& $max \\\\\n";
#}
#if($counter > 0) { $avg = $sum / $counter; } print FILE_CNT_TABLE "
#else { $avg = 0; } \\hline
#$avg = format_bytes($avg); \\end{tabular}
#$max = format_bytes($max); ";
#print TABLES "created files \& $counter \& $avg \& $max \\\\\n"; close(FILE_CNT_TABLE);
#
#print TABLES "
#\\hline
#\\end{tabular}
#";
#close(TABLES);
# generate per filesystem data # generate per filesystem data
open(FS_TABLE, ">$tmp_dir/fs-data-table.tex") || die("error opening output files:$!\n"); open(FS_TABLE, ">$tmp_dir/fs-data-table.tex") || die("error opening output files:$!\n");
...@@ -829,7 +820,7 @@ foreach $key (sort { $hash_files{$b}{'slowest_time'} <=> $hash_files{$a}{'slowes ...@@ -829,7 +820,7 @@ foreach $key (sort { $hash_files{$b}{'slowest_time'} <=> $hash_files{$a}{'slowes
my $vb = sprintf("%.3g", sqrt($hash_files{$key}{'variance_bytes'})); my $vb = sprintf("%.3g", sqrt($hash_files{$key}{'variance_bytes'}));
my $fast_bytes = format_bytes($hash_files{$key}{'fastest_bytes'}); my $fast_bytes = format_bytes($hash_files{$key}{'fastest_bytes'});
my $slow_bytes = format_bytes($hash_files{$key}{'slowest_bytes'}); my $slow_bytes = format_bytes($hash_files{$key}{'slowest_bytes'});
my $name = encode('latex', $hash_files{$key}{'name'}); my $name = encode('latex', "..." . substr($hash_files{$key}{'name'}, -12));
print VAR_TABLE " print VAR_TABLE "
$name \& $name \&
...@@ -905,7 +896,7 @@ my $cms = $cumul_meta_shared / $nprocs; ...@@ -905,7 +896,7 @@ my $cms = $cumul_meta_shared / $nprocs;
# do any extra work needed for plotting mpi-io graphs # do any extra work needed for plotting mpi-io graphs
if (defined $summary{MPIIO_INDEP_OPENS}) if (defined $summary{MPIIO_INDEP_OPENS})
{ {
system "$gnuplot -e \"data_file='mpiio-access-hist.dat'; graph_title='MPI-IO Access Sizes {\262}'; \\ system "$gnuplot -e \"data_file='mpiio-access-hist.dat'; graph_title='MPI-IO Access Sizes {/Times-Bold=32 \263}'; \\
output_file='mpiio-access-hist.eps'\" access-hist-eps.gplt"; output_file='mpiio-access-hist.eps'\" access-hist-eps.gplt";
system "$epstopdf mpiio-access-hist.eps"; system "$epstopdf mpiio-access-hist.eps";
...@@ -990,9 +981,9 @@ system "$mv $tmp_dir/summary.pdf $output_file"; ...@@ -990,9 +981,9 @@ system "$mv $tmp_dir/summary.pdf $output_file";
sub process_file_record sub process_file_record
{ {
my $rank = $_[0]; my $hash = $_[0];
my $hash = $_[1]; my(%file_record) = %{$_[1]};
my(%file_record) = %{$_[2]}; my $rank = $file_record{'RANK'};
if($file_record{'POSIX_OPENS'} == 0 && if($file_record{'POSIX_OPENS'} == 0 &&
$file_record{'POSIX_FOPENS'} == 0 && $file_record{'POSIX_FOPENS'} == 0 &&
......
...@@ -5,6 +5,8 @@ ...@@ -5,6 +5,8 @@
\usepackage{lastpage} \usepackage{lastpage}
\usepackage{subfigure} \usepackage{subfigure}
\usepackage{multirow} \usepackage{multirow}
\usepackage{threeparttable}
% %
% GET THE MARGINS RIGHT, THE UGLY WAY % GET THE MARGINS RIGHT, THE UGLY WAY
% %
...@@ -38,11 +40,11 @@ ...@@ -38,11 +40,11 @@
}\\ }\\
\subfigure \subfigure
{ {
\includegraphics[scale=0.8]{time-summary.pdf} \includegraphics[scale=0.75]{time-summary.pdf}
} }
\subfigure \subfigure
{ {
\includegraphics[scale=0.8]{op-counts.pdf} \includegraphics[scale=0.75]{op-counts.pdf}
} }
\end{figure*} \end{figure*}
...@@ -50,12 +52,12 @@ ...@@ -50,12 +52,12 @@
\centering \centering
\subfigure \subfigure
{ {
\includegraphics[scale=0.8]{posix-access-hist.pdf} \includegraphics[scale=0.75]{posix-access-hist.pdf}
} }
\ifdefined\inclmpiio \ifdefined\inclmpiio
\subfigure \subfigure
{ {
\includegraphics[scale=0.8]{mpiio-access-hist.pdf} \includegraphics[scale=0.75]{mpiio-access-hist.pdf}
} }
\fi \fi
\end{figure*} \end{figure*}
...@@ -66,10 +68,10 @@ ...@@ -66,10 +68,10 @@
{ {
\input{access-table.tex} \input{access-table.tex}
} }
%\subfigure \subfigure
%{ {
% \input{file-count-table} \input{file-count-table.tex}
%} }
\end{figure*} \end{figure*}
\begin{figure*}[!h] \begin{figure*}[!h]
...@@ -102,6 +104,10 @@ ...@@ -102,6 +104,10 @@
{