darshan-convert-logs.pl 1.97 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
#!/usr/bin/perl -w
#
#  (C) 2011 by Argonne National Laboratory.
#      See COPYRIGHT in top-level directory.
#
# Convert a list of logs and add metadata from another file.
#
# List of Logs: logfile path, one per line.
#	/path/to/log1
#	/path/to/log2
#
# Metadata: List of jobids with additional metadata, one per line, items are
#           tab separated.
#	jobid1	key1=val1	key2=val2
#	jobid2	key1=val1	key2=val2
#

my $darshan_convert = "./darshan-convert";

sub load_annotations($$)
{
    my $fname = shift;
    my $ref = shift;
    my $line;

    open(FH, "<$fname") or die("Failed to open annotations: $fname\n");
    while($line=<FH>)
    {
        my ($jobid, $annotation) = split(/\t/,$line,2);
        chomp($annotation);
        $ref->{$jobid} = $annotation;
    }
    close(FH);
}

sub main()
{
    my $annotation_list;
    my $logfile_list;
    my $hash_key;
    my $output_path;
    my $logfile;
    my $ref = {};

    $hash_key=$ARGV[0];
    $annotation_list=$ARGV[1];
    $logfile_list=$ARGV[2];
    $output_path=$ARGV[3];

    load_annotations($annotation_list, $ref);

    open(LOGFILES, "<$logfile_list") or die("Can't open: $logfile_list");
    while($logfile=<LOGFILES>)
    {
        chomp($logfile);
        if ($logfile =~ /_id(\d+)_/)
        {
            my $jobid;
            my $annotation;
            my $hashed_fname;
            my @args;

            $jobid = $1;
            $annotation = $ref->{$jobid};

            $hashed_fname = `./jenkins --64 --key $hash_key $logfile`;
            chomp($hashed_fname);

            @args = ("$darshan_convert",
                     "--obfuscate",
                     "--key=$hash_key",
                     "--annotate=$annotation",
                     "$logfile",
                     "$output_path/$hashed_fname.gz");
            system(@args);
            print("$hashed_fname\t$logfile\n");
        }
        else
        {
            print("Invalid logfile name: $logfile\n")
        }
    }

    return 0;
}

#
# Main
#
&main