darshan-convert-logs.pl 2.73 KB
Newer Older
1
#!/usr/bin/perl -w
Shane Snyder's avatar
Shane Snyder committed
2

3
#
Shane Snyder's avatar
Shane Snyder committed
4 5
# Copyright (C) 2015 University of Chicago.
# See COPYRIGHT notice in top-level directory.
6
#
Shane Snyder's avatar
Shane Snyder committed
7

8 9
# Convert a list of logs and add metadata from another file.
#
10 11 12
# List of Logs: logfile path, one per line.  This can be generated with a 
#           command like "find <path> -name "*.gz"".  Make sure that the
#           resulting list does not include .partial files.
13 14 15 16 17 18 19 20 21 22
#	/path/to/log1
#	/path/to/log2
#
# Metadata: List of jobids with additional metadata, one per line, items are
#           tab separated.
#	jobid1	key1=val1	key2=val2
#	jobid2	key1=val1	key2=val2
#

my $darshan_convert = "./darshan-convert";
23
my $jenkins_hash_gen = "./jenkins-hash-gen";
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66

sub load_annotations($$)
{
    my $fname = shift;
    my $ref = shift;
    my $line;

    open(FH, "<$fname") or die("Failed to open annotations: $fname\n");
    while($line=<FH>)
    {
        my ($jobid, $annotation) = split(/\t/,$line,2);
        chomp($annotation);
        $ref->{$jobid} = $annotation;
    }
    close(FH);
}

sub main()
{
    my $annotation_list;
    my $logfile_list;
    my $hash_key;
    my $output_path;
    my $logfile;
    my $ref = {};

    $hash_key=$ARGV[0];
    $annotation_list=$ARGV[1];
    $logfile_list=$ARGV[2];
    $output_path=$ARGV[3];

    load_annotations($annotation_list, $ref);

    open(LOGFILES, "<$logfile_list") or die("Can't open: $logfile_list");
    while($logfile=<LOGFILES>)
    {
        chomp($logfile);
        if ($logfile =~ /_id(\d+)_/)
        {
            my $jobid;
            my $annotation;
            my $hashed_fname;
            my @args;
67 68 69 70 71
            my $year;
            my $month;
            my $day;
            my $logname;
            my $rc;
72 73 74 75

            $jobid = $1;
            $annotation = $ref->{$jobid};

76 77 78 79 80 81 82
            if ($logfile =~ /\/(\d+)\/(\d+)\/(\d+)\/([0-9a-zA-Z\-_\.]+)/)
            {
                $year = $1;
                $month = $2;
                $day = $3;
                $logname = $4;
            }
83
            $hashed_fname = `$jenkins_hash_gen --64 --key $hash_key $logname`;
84 85 86
            chomp($hashed_fname);

            @args = ("$darshan_convert",
87
                     "--bzip2",
88 89 90
                     "--obfuscate",
                     "--key=$hash_key",
                     "--annotate=$annotation",
91
                     "--reset-md",
92
                     "$logfile",
93
                     "$output_path/$year/$month/$day/$hashed_fname.darshan");
94 95 96 97 98 99 100
            $rc = system(@args);
            if ($rc) {
                print("$hashed_fname\t$logfile:failed:$rc\n");
            }
            else {
                print("$hashed_fname\t$logfile\n");
            }
101 102 103 104 105 106 107 108 109 110 111 112 113 114
        }
        else
        {
            print("Invalid logfile name: $logfile\n")
        }
    }

    return 0;
}

#
# Main
#
&main