darshan-job-summary.pl 31 KB
Newer Older
1
#!/usr/bin/perl -w
2 3 4 5 6
#
#  (C) 2009 by Argonne National Laboratory.
#      See COPYRIGHT in top-level directory.
#

7 8
use FindBin;
use lib "$FindBin::Bin/../lib/";
9 10
use TeX::Encode;
use Encode;
11 12
use File::Temp qw/ tempdir /;
use Cwd;
13 14
use Getopt::Long;
use English;
15
use Number::Bytes::Human qw(format_bytes);
16 17 18 19

my $gnuplot = "";

my $tmp_dir = tempdir( CLEANUP => 1 );
20 21 22
#my $tmp_dir = tempdir( CLEANUP => 0 );
#print "tmp dir: $tmp_dir\n";

23
my $orig_dir = getcwd;
24 25
my $output_file = "summary.pdf";
my $input_file = "";
26 27
my %access_hash = ();
my @access_size = ();
28
my %hash_files = ();
29

30 31
process_args();

32 33 34 35 36 37 38 39 40 41 42 43 44
# find a darshan-parser executable somewhere
my $darshan_parser = "";
if(-x "$FindBin::Bin/../bin/darshan-parser")
{
    $darshan_parser = "$FindBin::Bin/../bin/darshan-parser";
}
else
{
    $darshan_parser = "darshan-parser";
}

open(TRACE, "$darshan_parser $input_file |") || die("Can't execute \"$darshan_parser $input_file\": $!\n");

45 46
open(FA_READ, ">$tmp_dir/file-access-read.dat") || die("error opening output file: $!\n");
open(FA_WRITE, ">$tmp_dir/file-access-write.dat") || die("error opening output file: $!\n");
47 48
open(FA_READ_SH, ">$tmp_dir/file-access-read-sh.dat") || die("error opening output file: $!\n");
open(FA_WRITE_SH, ">$tmp_dir/file-access-write-sh.dat") || die("error opening output file: $!\n");
49 50 51

my $last_read_start = 0;
my $last_write_start = 0;
52

53 54 55 56 57 58 59 60 61 62 63 64
my $cumul_read_indep = 0;
my $cumul_read_bytes_indep = 0;

my $cumul_write_indep = 0;
my $cumul_write_bytes_indep = 0;

my $cumul_read_shared = 0;
my $cumul_read_bytes_shared = 0;

my $cumul_write_shared = 0;
my $cumul_write_bytes_shared = 0;

65 66 67
my $cumul_meta_shared = 0;
my $cumul_meta_indep = 0;

68 69 70 71 72
my $first_data_line = 1;
my $current_rank = 0;
my $current_hash = 0;
my %file_record_hash = ();

73 74
my %fs_data = ();

75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
while ($line = <TRACE>) {
    chop($line);
    
    if ($line =~ /^\s*$/) {
        # ignore blank lines
    }
    elsif ($line =~ /^#/) {
	if ($line =~ /^# exe: /) {
	    ($junk, $cmdline) = split(':', $line, 2);
            # add escape characters if needed for special characters in
            # command line
            $cmdline = encode('latex', $cmdline);
	}
	if ($line =~ /^# nprocs: /) {
	    ($junk, $nprocs) = split(':', $line, 2);
	    $procreads[$nprocs] = 0;
	}
	if ($line =~ /^# run time: /) {
	    ($junk, $runtime) = split(':', $line, 2);
	}
	if ($line =~ /^# start_time: /) {
	    ($junk, $starttime) = split(':', $line, 2);
	}
	if ($line =~ /^# uid: /) {
	    ($junk, $uid) = split(':', $line, 2);
	}
101 102 103
        if ($line =~ /^# jobid: /) {
	    ($junk, $jobid) = split(':', $line, 2);
        }
104 105
    }
    else {
106
        # parse line
107
	@fields = split(/[\t ]+/, $line);
108 109 110 111 112 113 114 115 116 117

        # is this our first piece of data?
        if($first_data_line)
        {
            $current_rank = $fields[0];
            $current_hash = $fields[1];
            $first_data_line = 0;
        }

        # is this a new file record?
118
        if($fields[0] != $current_rank || $fields[1] != $current_hash)
119 120
        {
            # process previous record
121
            process_file_record($current_rank, $current_hash, \%file_record_hash);
122 123 124 125 126 127 128 129 130

            # reset variables for next record 
            $current_rank = $fields[0];
            $current_hash = $fields[1];
            %file_record_hash = ();
        }

        $file_record_hash{$fields[2]} = $fields[3];

131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
	$summary{$fields[2]} += $fields[3];

	# record per-process POSIX read count
	if ($fields[2] eq "CP_POSIX_READS") {
	    if ($fields[0] == -1) {
		$procreads[$nprocs] += $fields[3];
	    }
	    else {
		$procreads[$fields[0]] += $fields[3];
	    }
	}

	# record per-proces POSIX write count
	if ($fields[2] eq "CP_POSIX_WRITES") {
	    if ($fields[0] == -1) {
		$procwrites[$nprocs] += $fields[3];
	    }
	    else {
		$procwrites[$fields[0]] += $fields[3];
	    }
	}

153
        # seperate accumulators for independent and shared reads and writes
Philip Carns's avatar
Philip Carns committed
154
        if ($fields[2] eq "CP_F_POSIX_READ_TIME" && $fields[0] == -1){
155 156
            $cumul_read_shared += $fields[3];
        }
Philip Carns's avatar
Philip Carns committed
157
        if ($fields[2] eq "CP_F_POSIX_READ_TIME" && $fields[0] != -1){
158 159
            $cumul_read_indep += $fields[3];
        }
Philip Carns's avatar
Philip Carns committed
160
        if ($fields[2] eq "CP_F_POSIX_WRITE_TIME" && $fields[0] == -1){
161 162
            $cumul_write_shared += $fields[3];
        }
Philip Carns's avatar
Philip Carns committed
163
        if ($fields[2] eq "CP_F_POSIX_WRITE_TIME" && $fields[0] != -1){
164 165 166
            $cumul_write_indep += $fields[3];
        }

167 168 169 170 171 172 173
        if ($fields[2] eq "CP_F_POSIX_META_TIME" && $fields[0] == -1){
            $cumul_meta_shared += $fields[3];
        }
        if ($fields[2] eq "CP_F_POSIX_META_TIME" && $fields[0] != -1){
            $cumul_meta_indep += $fields[3];
        }

174 175 176 177 178 179
        if (($fields[2] eq "CP_BYTES_READ") or
            ($fields[2] eq "CP_BYTES_WRITTEN") and
            not defined($fs_data{$fields[5]}))
        {
            $fs_data{$fields[5]} = [0,0];
        }
180

Philip Carns's avatar
Philip Carns committed
181
        if ($fields[2] eq "CP_BYTES_READ" && $fields[0] == -1){
182
            $cumul_read_bytes_shared += $fields[3];
183
            $fs_data{$fields[5]}->[0] += $fields[3];
184
        }
Philip Carns's avatar
Philip Carns committed
185
        if ($fields[2] eq "CP_BYTES_READ" && $fields[0] != -1){
186
            $cumul_read_bytes_indep += $fields[3];
187
            $fs_data{$fields[5]}->[0] += $fields[3];
188
        }
Philip Carns's avatar
Philip Carns committed
189
        if ($fields[2] eq "CP_BYTES_WRITTEN" && $fields[0] == -1){
190
            $cumul_write_bytes_shared += $fields[3];
191
            $fs_data{$fields[5]}->[1] += $fields[3];
192
        }
Philip Carns's avatar
Philip Carns committed
193
        if ($fields[2] eq "CP_BYTES_WRITTEN" && $fields[0] != -1){
194
            $cumul_write_bytes_indep += $fields[3];
195
            $fs_data{$fields[5]}->[1] += $fields[3];
196 197
        }

198 199 200 201 202 203 204 205 206 207 208 209 210 211
        # record start and end of reads and writes

        if ($fields[2] eq "CP_F_READ_START_TIMESTAMP") {
            # store until we find the end
            # adjust for systems that give absolute time stamps
            $last_read_start = $fields[3];
        }
        if ($fields[2] eq "CP_F_READ_END_TIMESTAMP" && $fields[3] != 0) {
            # assume we got the read start already 
            my $xdelta = $fields[3] - $last_read_start;
            # adjust for systems that have absolute time stamps 
            if($last_read_start > $starttime) {
                $last_read_start -= $starttime;
            }
212 213 214 215 216 217
            if($fields[0] == -1){
                print FA_READ_SH "$last_read_start\t0\t$xdelta\t0\n";
            }
            else{
                print FA_READ "$last_read_start\t$fields[0]\t$xdelta\t0\n";
            }
218 219 220 221 222 223 224 225 226 227 228 229
        }
        if ($fields[2] eq "CP_F_WRITE_START_TIMESTAMP") {
            # store until we find the end
            $last_write_start = $fields[3];
        }
        if ($fields[2] eq "CP_F_WRITE_END_TIMESTAMP" && $fields[3] != 0) {
            # assume we got the write start already 
            my $xdelta = $fields[3] - $last_write_start;
            # adjust for systems that have absolute time stamps 
            if($last_write_start > $starttime) {
                $last_write_start -= $starttime;
            }
230 231 232 233 234 235
            if($fields[0] == -1){
                print FA_WRITE_SH "$last_write_start\t0\t$xdelta\t0\n";
            }
            else{
                print FA_WRITE "$last_write_start\t$fields[0]\t$xdelta\t0\n";
            }
236 237
        }

238 239
        if ($fields[2] =~ /^CP_ACCESS(.)_ACCESS/) {
            $access_size[$1] = $fields[3];
240
        }
241 242 243 244
        if ($fields[2] =~ /^CP_ACCESS(.)_COUNT/) {
            my $tmp_access_size = $access_size[$1];
            if(defined $access_hash{$tmp_access_size}){
                $access_hash{$tmp_access_size} += $fields[3];
245
            }
246 247
            else{
                $access_hash{$tmp_access_size} = $fields[3];
248 249 250 251 252
            }
        }
    }
}

253
# process last file record
254
process_file_record($current_rank, $current_hash, \%file_record_hash);
255
close(TRACE) || die "darshan-parser failure: $! $?";
256

257 258 259 260
# Fudge one point at the end to make xrange match in read and write plots.
# For some reason I can't get the xrange command to work.  -Phil
print FA_READ "$runtime\t-1\t0\t0\n";
print FA_WRITE "$runtime\t-1\t0\t0\n";
261 262
print FA_READ_SH "$runtime\t0\t0\t0\n";
print FA_WRITE_SH "$runtime\t0\t0\t0\n";
263 264
close(FA_READ);
close(FA_WRITE);
265 266
close(FA_READ_SH);
close(FA_WRITE_SH);
267

268
# counts of operations
269
open(COUNTS, ">$tmp_dir/counts.dat") || die("error opening output file: $!\n");
270 271
print COUNTS "# P=POSIX, MI=MPI-IO indep., MC=MPI-IO coll., R=read, W=write\n";
print COUNTS "# PR, MIR, MCR, PW, MIW, MCW, Popen, Pseek, Pstat\n";
272
my $total_syncs = $summary{CP_POSIX_FSYNCS} + $summary{CP_POSIX_FDSYNCS};
273 274 275 276 277 278 279 280
print COUNTS "Read, ", $summary{CP_POSIX_READS}, ", ",
    $summary{CP_INDEP_READS}, ", ", $summary{CP_COLL_READS}, "\n",
    "Write, ", $summary{CP_POSIX_WRITES}, ", ", 
    $summary{CP_INDEP_WRITES}, ", ", $summary{CP_COLL_WRITES}, "\n",
    "Open, ", $summary{CP_POSIX_OPENS}, ", ", $summary{CP_INDEP_OPENS},", ",
    $summary{CP_COLL_OPENS}, "\n",
    "Stat, ", $summary{CP_POSIX_STATS}, ", 0, 0\n",
    "Seek, ", $summary{CP_POSIX_SEEKS}, ", 0, 0\n",
281 282
    "Mmap, ", $summary{CP_POSIX_MMAPS}, ", 0, 0\n",
    "Fsync, ", $total_syncs, ", 0, 0\n";
283 284 285
close COUNTS;

# histograms of reads and writes
286
open (HIST, ">$tmp_dir/hist.dat") || die("error opening output file: $!\n");
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
print HIST "# size_range read write\n";
print HIST "0-100, ", $summary{CP_SIZE_READ_0_100}, ", ",
                 $summary{CP_SIZE_WRITE_0_100}, "\n";
print HIST "101-1K, ", $summary{CP_SIZE_READ_100_1K}, ", ",
                 $summary{CP_SIZE_WRITE_100_1K}, "\n";
print HIST "1K-10K, ", $summary{CP_SIZE_READ_1K_10K}, ", ",
                 $summary{CP_SIZE_WRITE_1K_10K}, "\n";
print HIST "10K-100K, ", $summary{CP_SIZE_READ_10K_100K}, ", ",
                 $summary{CP_SIZE_WRITE_10K_100K}, "\n";
print HIST "100K-1M, ", $summary{CP_SIZE_READ_100K_1M}, ", ",
                 $summary{CP_SIZE_WRITE_100K_1M}, "\n";
print HIST "1M-4M, ", $summary{CP_SIZE_READ_1M_4M}, ", ",
                 $summary{CP_SIZE_WRITE_1M_4M}, "\n";
print HIST "4M-10M, ", $summary{CP_SIZE_READ_4M_10M}, ", ",
                 $summary{CP_SIZE_WRITE_4M_10M}, "\n";
print HIST "10M-100M, ", $summary{CP_SIZE_READ_10M_100M}, ", ",
                 $summary{CP_SIZE_WRITE_10M_100M}, "\n";
print HIST "100M-1G, ", $summary{CP_SIZE_READ_100M_1G}, ", ",
                 $summary{CP_SIZE_WRITE_100M_1G}, "\n";
print HIST "1G+, ", $summary{CP_SIZE_READ_1G_PLUS}, ", ",
                 $summary{CP_SIZE_WRITE_1G_PLUS}, "\n";
close HIST;

# sequential and consecutive accesses
311
open (PATTERN, ">$tmp_dir/pattern.dat") || die("error opening output file: $!\n");
312 313 314 315 316 317 318 319
print PATTERN "# op total sequential consecutive\n";
print PATTERN "Read, ", $summary{CP_POSIX_READS}, ", ",
    $summary{CP_SEQ_READS}, ", ", $summary{CP_CONSEC_READS}, "\n";
print PATTERN "Write, ", $summary{CP_POSIX_WRITES}, ", ",
    $summary{CP_SEQ_WRITES}, ", ", $summary{CP_CONSEC_WRITES}, "\n";
close PATTERN;

# aligned I/O
320
open (ALIGN, ">$tmp_dir/align.dat") || die("error opening output file: $!\n");
321 322 323 324 325 326
print ALIGN "# total unaligned_mem unaligned_file align_mem align_file\n";
print ALIGN $summary{CP_POSIX_READS} + $summary{CP_POSIX_WRITES}, ", ",
    $summary{CP_MEM_NOT_ALIGNED}, ", ", $summary{CP_FILE_NOT_ALIGNED}, "\n";
close ALIGN;

# MPI types
327
open (TYPES, ">$tmp_dir/types.dat") || die("error opening output file: $!\n");
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374
print TYPES "# type use_count\n";
print TYPES "Named, ", $summary{CP_COMBINER_NAMED}, "\n";
print TYPES "Dup, ", $summary{CP_COMBINER_DUP}, "\n";
print TYPES "Contig, ", $summary{CP_COMBINER_CONTIGUOUS}, "\n";
print TYPES "Vector, ", $summary{CP_COMBINER_VECTOR}, "\n";
print TYPES "HvecInt, ", $summary{CP_COMBINER_HVECTOR_INTEGER}, "\n";
print TYPES "Hvector, ", $summary{CP_COMBINER_HVECTOR}, "\n";
print TYPES "Indexed, ", $summary{CP_COMBINER_INDEXED}, "\n";
print TYPES "HindInt, ", $summary{CP_COMBINER_HINDEXED_INTEGER}, "\n";
print TYPES "Hindexed, ", $summary{CP_COMBINER_HINDEXED}, "\n";
print TYPES "IndBlk, ", $summary{CP_COMBINER_INDEXED_BLOCK}, "\n";
print TYPES "StructInt, ", $summary{CP_COMBINER_STRUCT_INTEGER}, "\n";
print TYPES "Struct, ", $summary{CP_COMBINER_STRUCT}, "\n";
print TYPES "Subarray, ", $summary{CP_COMBINER_SUBARRAY}, "\n";
print TYPES "Darray, ", $summary{CP_COMBINER_DARRAY}, "\n";
print TYPES "F90Real, ", $summary{CP_COMBINER_F90_REAL}, "\n";
print TYPES "F90Complex, ", $summary{CP_COMBINER_F90_COMPLEX}, "\n";
print TYPES "F90Int, ", $summary{CP_COMBINER_F90_INTEGER}, "\n";
print TYPES "Resized, ", $summary{CP_COMBINER_RESIZED}, "\n";
close TYPES;

# generate histogram of process I/O counts
#
# NOTE: NEED TO FILL IN ACTUAL WRITE DATA!!!
#
$minprocread = (defined $procreads[0]) ? $procreads[0] : 0;
$maxprocread = (defined $procreads[0]) ? $procreads[0] : 0;
for ($i=1; $i < $nprocs; $i++) {
    $rdi = (defined $procreads[$i]) ? $procreads[$i] : 0;
    $minprocread = ($rdi > $minprocread) ? $minprocread : $rdi;
    $maxprocread = ($rdi < $maxprocread) ? $maxprocread : $rdi;
}
$minprocread += $procreads[$nprocs];
$maxprocread += $procreads[$nprocs];
# print "$minprocread $maxprocread\n";

@bucket = ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 );

for ($i=0; $i < $nprocs; $i++) {
    $mysize = ((defined $procreads[$i]) ? $procreads[$i] : 0) +
	$procreads[$nprocs];
    $mysize -= $minprocread;
    $mybucket = ($mysize > 0) ?
	(($mysize * 10) / ($maxprocread - $minprocread)) : 0;
    $bucket[$mybucket]++;
}

375
open(IODIST, ">$tmp_dir/iodist.dat") || die("error opening output file: $!\n");
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
print IODIST "# bucket n_procs_rd n_procs_wr\n";
print IODIST "# NOTE: WRITES ARE A COPY OF READS FOR NOW!!!\n";

$bucketsize = $maxprocread - $minprocread / 10;
# TODO: do writes also, is dropping a 0 in for now
for ($i=0; $i < 10; $i++) {
    print IODIST $bucketsize * $i + $minprocread, "-",
    $bucketsize * ($i+1) + $minprocread, ", ", $bucket[$i], ", 0\n";
}
close IODIST;

# generate title for summary
($executable, $junk) = split(' ', $cmdline, 2);
@parts = split('/', $executable);
$cmd = $parts[$#parts];

@timearray = localtime($starttime);
$year = $timearray[5] + 1900;
$mon = $timearray[4] + 1;
$mday = $timearray[3];

397
open(TITLE, ">$tmp_dir/title.tex") || die("error opening output file:$!\n");
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
print TITLE "
\\rhead{\\thepage\\ of \\pageref{LastPage}}
\\chead[
\\large $cmd ($mon/$mday/$year)
]
{
\\large $cmd ($mon/$mday/$year)
}
\\cfoot[
\\scriptsize{$cmdline}
]
{
\\scriptsize{$cmdline}
}
";
close TITLE;

415
open(TABLES, ">$tmp_dir/job-table.tex") || die("error opening output file:$!\n");
416
print TABLES "
417
\\begin{tabular}{|p{.47\\columnwidth}|p{.35\\columnwidth}|p{.47\\columnwidth}|p{.6\\columnwidth}|}
418
\\hline
419
jobid: $jobid \& uid: $uid \& nprocs: $nprocs \& runtime: $runtime seconds\\\\
420 421 422 423 424
\\hline
\\end{tabular}
";
close TABLES;

425
open(TABLES, ">$tmp_dir/access-table.tex") || die("error opening output file:$!\n");
426
print TABLES "
Philip Carns's avatar
Philip Carns committed
427
\\begin{tabular}{r|r}
428
\\multicolumn{2}{c}{ } \\\\
429
\\multicolumn{2}{c}{Most Common Access Sizes} \\\\
430 431 432
\\hline
access size \& count \\\\
\\hline
Philip Carns's avatar
Philip Carns committed
433
\\hline
434 435
";

436 437
# sort access sizes (descending)
my $i = 0;
438 439
foreach $value (sort {$access_hash{$b} <=> $access_hash{$a} } keys %access_hash) {
    if($i == 4) {
440 441 442 443 444 445 446 447 448
        last;
    }
    if($access_hash{$value} == 0) {
        last;
    }
    print TABLES "$value \& $access_hash{$value} \\\\\n";
    $i++;
}

449 450 451 452 453 454
print TABLES "
\\hline
\\end{tabular}
";
close TABLES;

Philip Carns's avatar
Philip Carns committed
455 456 457
open(TABLES, ">$tmp_dir/file-count-table.tex") || die("error opening output file:$!\n");
print TABLES "
\\begin{tabular}{r|r|r|r}
458
\\multicolumn{4}{c}{ } \\\\
Philip Carns's avatar
Philip Carns committed
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
\\multicolumn{4}{c}{File Count Summary} \\\\
\\hline
type \& number of files \& avg. size \& max size \\\\
\\hline
\\hline
";
my $counter;
my $sum;
my $max;
my $key;
my $avg;

$counter = 0;
$sum = 0;
$max = 0;
foreach $key (keys %hash_files) {
    $counter++;
    if($hash_files{$key}{'min_open_size'} >
        $hash_files{$key}{'max_size'})
    {
        $sum += $hash_files{$key}{'min_open_size'};
        if($hash_files{$key}{'min_open_size'} > $max)
        {
            $max = $hash_files{$key}{'min_open_size'};
        }
    }
    else
    {
        $sum += $hash_files{$key}{'max_size'};
        if($hash_files{$key}{'max_size'} > $max)
        {
            $max = $hash_files{$key}{'max_size'};
        }
    }
}
494 495
if($counter > 0) { $avg = $sum / $counter; }
else { $avg = 0; }
496 497
$avg = format_bytes($avg);
$max = format_bytes($max);
Philip Carns's avatar
Philip Carns committed
498 499
print TABLES "total opened \& $counter \& $avg \& $max \\\\\n";

500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629
$counter = 0;
$sum = 0;
$max = 0;
foreach $key (keys %hash_files) {
    if($hash_files{$key}{'was_read'} && !($hash_files{$key}{'was_written'}))
    {
        $counter++;
        if($hash_files{$key}{'min_open_size'} >
            $hash_files{$key}{'max_size'})
        {
            $sum += $hash_files{$key}{'min_open_size'};
            if($hash_files{$key}{'min_open_size'} > $max)
            {
                $max = $hash_files{$key}{'min_open_size'};
            }
        }
        else
        {
            $sum += $hash_files{$key}{'max_size'};
            if($hash_files{$key}{'max_size'} > $max)
            {
                $max = $hash_files{$key}{'max_size'};
            }
        }
    }
}
if($counter > 0) { $avg = $sum / $counter; }
else { $avg = 0; }
$avg = format_bytes($avg);
$max = format_bytes($max);
print TABLES "read-only files \& $counter \& $avg \& $max \\\\\n";

$counter = 0;
$sum = 0;
$max = 0;
foreach $key (keys %hash_files) {
    if(!($hash_files{$key}{'was_read'}) && $hash_files{$key}{'was_written'})
    {
        $counter++;
        if($hash_files{$key}{'min_open_size'} >
            $hash_files{$key}{'max_size'})
        {
            $sum += $hash_files{$key}{'min_open_size'};
            if($hash_files{$key}{'min_open_size'} > $max)
            {
                $max = $hash_files{$key}{'min_open_size'};
            }
        }
        else
        {
            $sum += $hash_files{$key}{'max_size'};
            if($hash_files{$key}{'max_size'} > $max)
            {
                $max = $hash_files{$key}{'max_size'};
            }
        }
    }
}
if($counter > 0) { $avg = $sum / $counter; }
else { $avg = 0; }
$avg = format_bytes($avg);
$max = format_bytes($max);
print TABLES "write-only files \& $counter \& $avg \& $max \\\\\n";

$counter = 0;
$sum = 0;
$max = 0;
foreach $key (keys %hash_files) {
    if($hash_files{$key}{'was_read'} && $hash_files{$key}{'was_written'})
    {
        $counter++;
        if($hash_files{$key}{'min_open_size'} >
            $hash_files{$key}{'max_size'})
        {
            $sum += $hash_files{$key}{'min_open_size'};
            if($hash_files{$key}{'min_open_size'} > $max)
            {
                $max = $hash_files{$key}{'min_open_size'};
            }
        }
        else
        {
            $sum += $hash_files{$key}{'max_size'};
            if($hash_files{$key}{'max_size'} > $max)
            {
                $max = $hash_files{$key}{'max_size'};
            }
        }
    }
}
if($counter > 0) { $avg = $sum / $counter; }
else { $avg = 0; }
$avg = format_bytes($avg);
$max = format_bytes($max);
print TABLES "read/write files \& $counter \& $avg \& $max \\\\\n";

$counter = 0;
$sum = 0;
$max = 0;
foreach $key (keys %hash_files) {
    if($hash_files{$key}{'was_written'} &&
        $hash_files{$key}{'min_open_size'} == 0 &&
        $hash_files{$key}{'max_size'} > 0)
    {
        $counter++;
        if($hash_files{$key}{'min_open_size'} >
            $hash_files{$key}{'max_size'})
        {
            $sum += $hash_files{$key}{'min_open_size'};
            if($hash_files{$key}{'min_open_size'} > $max)
            {
                $max = $hash_files{$key}{'min_open_size'};
            }
        }
        else
        {
            $sum += $hash_files{$key}{'max_size'};
            if($hash_files{$key}{'max_size'} > $max)
            {
                $max = $hash_files{$key}{'max_size'};
            }
        }
    }
}
if($counter > 0) { $avg = $sum / $counter; }
else { $avg = 0; }
$avg = format_bytes($avg);
$max = format_bytes($max);
print TABLES "created files \& $counter \& $avg \& $max \\\\\n";

Philip Carns's avatar
Philip Carns committed
630 631 632 633 634 635 636
print TABLES "
\\hline
\\end{tabular}
";
close(TABLES);


637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
#
# Generate Per Filesystem Data
#
open(TABLES, ">$tmp_dir/fs-data-table.tex") || die("error opening output files:$!\n");
print TABLES "
\\begin{tabular}{c|r|r|r|r}
\\multicolumn{5}{c}{ } \\\\
\\multicolumn{5}{c}{Data Transfer Per Filesystem} \\\\
\\hline
\\multirow{2}{*}{File System} \& \\multicolumn{2}{c}{Write} \\vline \& \\multicolumn{2}{c}{Read} \\\\
\\cline{2-5}
\& MiB \& Ratio \& MiB \& Ratio \\\\\
\\hline
\\hline
";
foreach $key (keys %fs_data)
{
    my $wr_total_mb = ($fs_data{$key}->[1] / (1024*1024));
    my $rd_total_mb = ($fs_data{$key}->[0] / (1024*1024));
    my $wr_total_rt = ($fs_data{$key}->[1] / ($cumul_write_bytes_shared + $cumul_write_bytes_indep));
    my $rd_total_rt = ($fs_data{$key}->[0] / ($cumul_read_bytes_shared + $cumul_read_bytes_indep));

    printf TABLES "%s \& %.5f \& %.5f \& %.5f \& %.5f \\\\\n",
        $key, $wr_total_mb, $wr_total_rt, $rd_total_mb, $rd_total_rt;
}
print TABLES "
\\hline
\\end{tabular}
";
close(TABLES);


669
open(TIME, ">$tmp_dir/time-summary.dat") || die("error opening output file:$!\n");
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
print TIME "# <type>, <app time>, <read>, <write>, <meta>\n";
print TIME "POSIX, ", ((($runtime * $nprocs - $summary{CP_F_POSIX_READ_TIME} -
    $summary{CP_F_POSIX_WRITE_TIME} -
    $summary{CP_F_POSIX_META_TIME})/($runtime * $nprocs)) * 100);
print TIME ", ", (($summary{CP_F_POSIX_READ_TIME}/($runtime * $nprocs))*100);
print TIME ", ", (($summary{CP_F_POSIX_WRITE_TIME}/($runtime * $nprocs))*100);
print TIME ", ", (($summary{CP_F_POSIX_META_TIME}/($runtime * $nprocs))*100), "\n";
print TIME "MPI-IO, ", ((($runtime * $nprocs - $summary{CP_F_MPI_READ_TIME} -
    $summary{CP_F_MPI_WRITE_TIME} -
    $summary{CP_F_MPI_META_TIME})/($runtime * $nprocs)) * 100);
print TIME ", ", (($summary{CP_F_MPI_READ_TIME}/($runtime * $nprocs))*100);
print TIME ", ", (($summary{CP_F_MPI_WRITE_TIME}/($runtime * $nprocs))*100);
print TIME ", ", (($summary{CP_F_MPI_META_TIME}/($runtime * $nprocs))*100), "\n";
close TIME;

685
# copy template files to tmp tmp_dir
Philip Carns's avatar
Philip Carns committed
686 687
system "cp $FindBin::Bin/../share/*.gplt $tmp_dir/";
system "cp $FindBin::Bin/../share/*.tex $tmp_dir/";
688

689 690 691 692 693 694 695
# generate template for file access plot (we have to set range)
my $ymax = $nprocs + 1;
open(FILEACC, ">$tmp_dir/file-access-read-eps.gplt") || die("error opening output file:$!\n");
print FILEACC "#!/usr/bin/gnuplot -persist

set terminal postscript eps color solid font \"Helvetica\" 18 size 10in,2.5in
set output \"file-access-read.eps\"
696 697
set ylabel \"MPI rank\"
set xlabel \"hours:minutes:seconds\"
698 699 700
set xdata time
set timefmt \"%s\"
set format x \"%H:%M:%S\"
701
set yrange [-1:$ymax]
Philip Carns's avatar
Philip Carns committed
702
set title \"Timespan from first to last read access on independent files\"
703 704 705
# the xrange doesn't work for some reason
#set xrange [0:$runtime]
#set ytics -1,1
706
set lmargin 4
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725

# color blindness work around
set style line 2 lc 3
set style line 3 lc 4
set style line 4 lc 5
set style line 5 lc 2
set style increment user

# lw 3 to make lines thicker...
# note that writes are slightly offset for better visibility
plot \"file-access-read.dat\" using 1:2:3:4 with vectors nohead filled notitle
";
close FILEACC;

open(FILEACC, ">$tmp_dir/file-access-write-eps.gplt") || die("error opening output file:$!\n");
print FILEACC "#!/usr/bin/gnuplot -persist

set terminal postscript eps color solid font \"Helvetica\" 18 size 10in,2.5in
set output \"file-access-write.eps\"
726 727
set ylabel \"MPI rank\"
set xlabel \"hours:minutes:seconds\"
728 729 730
set xdata time
set timefmt \"%s\"
set format x \"%H:%M:%S\"
Philip Carns's avatar
Philip Carns committed
731
set title \"Timespan from first to last write access on independent files\"
732
set yrange [-1:$ymax]
733 734 735
# the xrange doesn't work for some reason
# set xrange [0:$runtime]
#set ytics -1,1
736
set lmargin 4
737 738 739 740 741 742 743 744 745 746 747 748 749

# color blindness work around
set style line 2 lc 3
set style line 3 lc 4
set style line 4 lc 5
set style line 5 lc 2
set style increment user

# lw 3 to make lines thicker...
plot \"file-access-write.dat\" using 1:2:3:4 with vectors nohead filled lt 2 notitle
";
close FILEACC;

750 751 752 753 754 755 756 757 758 759
open(FILEACC, ">$tmp_dir/file-access-shared-eps.gplt") || die("error opening output file:$!\n");
print FILEACC "#!/usr/bin/gnuplot -persist

set terminal postscript eps color solid font \"Helvetica\" 18 size 10in,2.5in
set output \"file-access-shared.eps\"
set xlabel \"hours:minutes:seconds\"
set xdata time
set timefmt \"%s\"
set format x \"%H:%M:%S\"
unset ytics
760
set ylabel \"All processes\"
761 762 763
# the xrange doesn't work for some reason
# set xrange [0:$runtime]
set yrange [-1:1]
Philip Carns's avatar
Philip Carns committed
764
set title \"Timespan from first to last access on files shared by all processes\"
765
set lmargin 4
766 767 768 769 770 771 772 773 774 775 776 777 778

# color blindness work around
set style line 2 lc 3
set style line 3 lc 4
set style line 4 lc 5
set style line 5 lc 2
set style increment user

plot \"file-access-read-sh.dat\" using 1:2:3:4 with vectors nohead filled lw 10 title \"read\", \\
\"file-access-write-sh.dat\" using 1:((\$2)-.2):3:4 with vectors nohead filled lw 10 title \"write\"
";
close FILEACC;

779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
$cumul_read_indep /= $nprocs;
$cumul_read_bytes_indep /= $nprocs;
$cumul_read_bytes_indep /= 1048576.0;

$cumul_write_indep /= $nprocs;
$cumul_write_bytes_indep /= $nprocs;
$cumul_write_bytes_indep /= 1048576.0;

$cumul_read_shared /= $nprocs;
$cumul_read_bytes_shared /= $nprocs;
$cumul_read_bytes_shared /= 1048576.0;

$cumul_write_shared /= $nprocs;
$cumul_write_bytes_shared /= $nprocs;
$cumul_write_bytes_shared /= 1048576.0;

795 796 797
$cumul_meta_shared /= $nprocs;
$cumul_meta_indep /= $nprocs;

798
open(FILEACC, ">$tmp_dir/file-access-table.tex") || die("error opening output file:$!\n");
799
print FILEACC "
Philip Carns's avatar
Philip Carns committed
800 801 802 803
\\begin{tabular}{l|p{1.7in}r}
\\multicolumn{3}{c}{Average I/O per process} \\\\
\\hline
 \& Cumulative time spent in I/O functions (seconds) \& Amount of I/O (MB) \\\\
804 805
\\hline
\\hline
Philip Carns's avatar
Philip Carns committed
806 807
";

Philip Carns's avatar
Philip Carns committed
808 809 810 811 812
# printf to get consistent precision in output
printf(FILEACC "Independent reads \& \\multicolumn{1}{r}{%f} \& \\multicolumn{1}{r}{%f} \\\\", 
    $cumul_read_indep, $cumul_read_bytes_indep);
printf(FILEACC "Independent writes \& \\multicolumn{1}{r}{%f} \& \\multicolumn{1}{r}{%f} \\\\", 
    $cumul_write_indep, $cumul_write_bytes_indep);
813 814
printf(FILEACC "Independent metadata \& \\multicolumn{1}{r}{%f} \& \\multicolumn{1}{r}{N/A} \\\\", 
    $cumul_meta_indep);
Philip Carns's avatar
Philip Carns committed
815 816 817 818
printf(FILEACC "Shared reads \& \\multicolumn{1}{r}{%f} \& \\multicolumn{1}{r}{%f} \\\\", 
    $cumul_read_shared, $cumul_read_bytes_shared);
printf(FILEACC "Shared writes \& \\multicolumn{1}{r}{%f} \& \\multicolumn{1}{r}{%f} \\\\", 
    $cumul_write_shared, $cumul_write_bytes_shared);
819 820
printf(FILEACC "Shared metadata \& \\multicolumn{1}{r}{%f} \& \\multicolumn{1}{r}{N/A} \\\\", 
    $cumul_meta_shared);
Philip Carns's avatar
Philip Carns committed
821 822

print FILEACC "
823 824 825
\\hline
\\end{tabular}
";
826 827
close(FILEACC);

828 829 830 831 832 833 834 835 836 837 838 839
if(-x "$FindBin::Bin/gnuplot")
{
    $gnuplot = "$FindBin::Bin/gnuplot";
}
else
{
    $gnuplot = "gnuplot";
}

# move to tmp_dir
chdir $tmp_dir;

840
# execute gnuplot scripts
841
system "$gnuplot counts-eps.gplt";
842
system "epstopdf counts.eps";
843
system "$gnuplot hist-eps.gplt";
844
system "epstopdf hist.eps";
845
system "$gnuplot pattern-eps.gplt";
846
system "epstopdf pattern.eps";
847
system "$gnuplot time-summary-eps.gplt";
848
system "epstopdf time-summary.eps";
849 850 851 852
system "$gnuplot file-access-read-eps.gplt";
system "epstopdf file-access-read.eps";
system "$gnuplot file-access-write-eps.gplt";
system "epstopdf file-access-write.eps";
853 854
system "$gnuplot file-access-shared-eps.gplt";
system "epstopdf file-access-shared.eps";
855

856 857 858 859
#system "gnuplot align-pdf.gplt";
#system "gnuplot iodist-pdf.gplt";
#system "gnuplot types-pdf.gplt";

860 861 862 863
# generate summary PDF
system "pdflatex -halt-on-error summary.tex > latex.output";
system "pdflatex -halt-on-error summary.tex > latex.output2";

864 865 866 867
# get back out of tmp dir and grab results
chdir $orig_dir;
system "mv $tmp_dir/summary.pdf $output_file";

868 869
sub process_file_record
{
870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
    my $rank = $_[0];
    my $hash = $_[1];
    my(%file_record) = %{$_[2]};

    if($file_record{'CP_INDEP_OPENS'} == 0 &&
        $file_record{'CP_COLL_OPENS'} == 0 &&
        $file_record{'CP_POSIX_OPENS'} == 0)
    {
        # file wasn't really opened, just stat probably
        return;
    }

    # record smallest open time size reported by any rank
    if(!defined($hash_files{$hash}{'min_open_size'}) ||
        $hash_files{$hash}{'min_open_size'} > 
        $file_record{'CP_SIZE_AT_OPEN'})
    {
        $hash_files{$hash}{'min_open_size'} = 
            $file_record{'CP_SIZE_AT_OPEN'};
    }

    # record largest size that the file reached at any rank
    if(!defined($hash_files{$hash}{'max_size'}) ||
        $hash_files{$hash}{'max_size'} <  
        ($file_record{'CP_MAX_BYTE_READ'} + 1))
    {
        $hash_files{$hash}{'max_size'} = 
            $file_record{'CP_MAX_BYTE_READ'} + 1;
    }
    if(!defined($hash_files{$hash}{'max_size'}) ||
        $hash_files{$hash}{'max_size'} <  
        ($file_record{'CP_MAX_BYTE_WRITTEN'} + 1))
    {
        $hash_files{$hash}{'max_size'} = 
            $file_record{'CP_MAX_BYTE_WRITTEN'} + 1;
    }

    # make sure there is an initial value for read and write flags
    if(!defined($hash_files{$hash}{'was_read'}))
    {
        $hash_files{$hash}{'was_read'} = 0;
    }
    if(!defined($hash_files{$hash}{'was_written'}))
    {
        $hash_files{$hash}{'was_written'} = 0;
    }

    if($file_record{'CP_INDEP_OPENS'} > 0 ||
        $file_record{'CP_COLL_OPENS'} > 0)
    {
        # mpi file
        if($file_record{'CP_INDEP_READS'} > 0 ||
            $file_record{'CP_COLL_READS'} > 0 ||
            $file_record{'CP_SPLIT_READS'} > 0 ||
            $file_record{'CP_NB_READS'} > 0)
        {
            # data was read from the file
            $hash_files{$hash}{'was_read'} = 1;
        }
        if($file_record{'CP_INDEP_WRITES'} > 0 ||
            $file_record{'CP_COLL_WRITES'} > 0 ||
            $file_record{'CP_SPLIT_WRITES'} > 0 ||
            $file_record{'CP_NB_WRITES'} > 0)
        {
            # data was written to the file
            $hash_files{$hash}{'was_written'} = 1;
        }
    }
    else
    {
        # posix file
        if($file_record{'CP_POSIX_READS'} > 0 ||
            $file_record{'CP_POSIX_FREADS'} > 0)
        {
            # data was read from the file
            $hash_files{$hash}{'was_read'} = 1;
        }
        if($file_record{'CP_POSIX_WRITES'} > 0 ||
            $file_record{'CP_POSIX_FWRITES'} > 0)
        {
            # data was written to the file 
            $hash_files{$hash}{'was_written'} = 1;
        }
    }

    # TODO 
    # (detect mpi or posix and):
    # - sum meta time per rank for uniq files
    # - sum io time per rank for uniq files
    # - sum time from first open to last io for shared files
    # - sum meta time/nprocs for shared files
    # - sum io time/nprocs for shared files
    
    # TODO: ideas
    # graph time spent performing I/O per rank
    # for rank that spent the most time performing I/O:
    # - meta on ro files, meta on wo files, read time, write time
    # table with nfiles accessed, ro, wo, rw, created
968 969
}

970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
sub process_args
{
    use vars qw( $opt_help $opt_output );

    Getopt::Long::Configure("no_ignore_case", "bundling");
    GetOptions( "help",
        "output=s");

    if($opt_help)
    {
        print_help();
        exit(0);
    }

    if($opt_output)
    {
        $output_file = $opt_output;
    }

    # there should only be one remaining argument: the input file 
    if($#ARGV != 0)
    {
        print "Error: invalid arguments.\n";
        print_help();
        exit(1);
    }
    $input_file = $ARGV[0];

    return;
}

sub print_help
{
    print <<EOF;

Usage: $PROGRAM_NAME <options> input_file

    --help          Prints this help message
    --output        Specifies a file to write pdf output to
                    (defaults to ./summary.pdf)
Purpose:

1012 1013
    This script reads a Darshan output file generated by a job and
    generates a pdf file summarizing job behavior.
1014 1015 1016 1017

EOF
    return;
}