Perl script to iterate through a list of logs and convert them, adding annotations.

  ./darshan-convert-logs <key> <file_with_annotations> <file_with_log_names> <output directory>


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@440 3b7491f3-a168-0410-bf4b-c445ed680a29

Makefile.in

@@ -56,6 +56,9 @@ darshan-analyzer: darshan-analyzer.c darshan.h darshan-log-format.h darshan-logu
 darshan-log-params: darshan-log-params.c darshan-log-format.h
 	gcc $(CFLAGS) $(LIBS) $< -o $@
 
+jenkins: util/bin/jenkins.o lib/lookup3.o
+	gcc $(CFLAGS) $(LIBS) $< -o $@ lib/lookup3.o
+
 darshan-diff: darshan-diff.o darshan.h darshan-log-format.h darshan-logutils.o darshan-logutils.h
 	gcc $(CFLAGS) $(LIBS) $< darshan-logutils.o -o $@
 darshan-diff.o: darshan-diff.c

darshan-convert-logs.pl

+#!/usr/bin/perl -w
+#
+#  (C) 2011 by Argonne National Laboratory.
+#      See COPYRIGHT in top-level directory.
+#
+# Convert a list of logs and add metadata from another file.
+#
+# List of Logs: logfile path, one per line.
+#	/path/to/log1
+#	/path/to/log2
+#
+# Metadata: List of jobids with additional metadata, one per line, items are
+#           tab separated.
+#	jobid1	key1=val1	key2=val2
+#	jobid2	key1=val1	key2=val2
+#
+
+my $darshan_convert = "./darshan-convert";
+
+sub load_annotations($$)
+{
+    my $fname = shift;
+    my $ref = shift;
+    my $line;
+
+    open(FH, "<$fname") or die("Failed to open annotations: $fname\n");
+    while($line=<FH>)
+    {
+        my ($jobid, $annotation) = split(/\t/,$line,2);
+        chomp($annotation);
+        $ref->{$jobid} = $annotation;
+    }
+    close(FH);
+}
+
+sub main()
+{
+    my $annotation_list;
+    my $logfile_list;
+    my $hash_key;
+    my $output_path;
+    my $logfile;
+    my $ref = {};
+
+    $hash_key=$ARGV[0];
+    $annotation_list=$ARGV[1];
+    $logfile_list=$ARGV[2];
+    $output_path=$ARGV[3];
+
+    load_annotations($annotation_list, $ref);
+
+    open(LOGFILES, "<$logfile_list") or die("Can't open: $logfile_list");
+    while($logfile=<LOGFILES>)
+    {
+        chomp($logfile);
+        if ($logfile =~ /_id(\d+)_/)
+        {
+            my $jobid;
+            my $annotation;
+            my $hashed_fname;
+            my @args;
+
+            $jobid = $1;
+            $annotation = $ref->{$jobid};
+
+            $hashed_fname = `./jenkins --64 --key $hash_key $logfile`;
+            chomp($hashed_fname);
+
+            @args = ("$darshan_convert",
+                     "--obfuscate",
+                     "--key=$hash_key",
+                     "--annotate=$annotation",
+                     "$logfile",
+                     "$output_path/$hashed_fname.gz");
+            system(@args);
+            print("$hashed_fname\t$logfile\n");
+        }
+        else
+        {
+            print("Invalid logfile name: $logfile\n")
+        }
+    }
+
+    return 0;
+}
+
+#
+# Main
+#
+&main

util/bin/jenkins.c

+/*
+ *  (C) 2011 by Argonne National Laboratory.
+ *      See COPYRIGHT in top-level directory.
+ */
+
+/*
+ * Small utility to do Jenkins hashes
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <getopt.h>
+#include <string.h>
+
+#define TYPE_32 (1)
+#define TYPE_64 (2)
+
+extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
+extern void hashlittle2(const void *key, size_t length, uint32_t *pc, uint32_t *pb);
+
+void usage(char *progname)
+{
+    fprintf(stderr, "usage: %s [--32] [--64] [--key <key>] data\n", progname);
+    exit(1);
+}
+
+void parse_args (int argc, char **argv, int* type, int* key, char **data)
+{
+    int index;
+    static struct option long_opts[] =
+    {
+        {"32", 0, NULL, 'a'},
+        {"64", 0, NULL, 'b'},
+        {"key", 1, NULL, 'k'},
+        {"help",  0, NULL, 0}
+    };
+
+    while(1)
+    {
+        int c = getopt_long(argc, argv, "", long_opts, &index);
+
+        if (c == -1) break;
+
+        switch(c)
+        {
+            case 'a':
+                *type = TYPE_32;
+                break;
+            case 'b':
+                *type = TYPE_64;
+                break;
+            case 'k':
+                *key = atoi(optarg);
+                break;
+            case 0:
+            case '?':
+            default:
+                usage(argv[0]);
+                break;
+        }
+    }
+
+    if (optind + 1 == argc)
+    {
+        *data = argv[optind];
+    }
+    else
+    {
+       usage(argv[0]);
+    }
+
+    return;
+}
+
+int main (int argc, char **argv)
+{
+    int type = TYPE_32;
+    int key = 0;
+    char *data;
+    uint32_t hashed_u = 0;
+    uint32_t hashed_l = 0;
+    uint64_t hashed;
+
+    parse_args (argc, argv, &type, &key, &data);
+
+    switch(type)
+    {
+    case TYPE_64:
+        hashed_u = key;
+        hashlittle2(data, strlen(data), &hashed_u, &hashed_l);
+        break;
+    case TYPE_32:
+    default: 
+        hashed_l = darshan_hashlittle(data, strlen(data), key);
+        break;
+    }
+
+    hashed = ((uint64_t)hashed_u << 32) | hashed_l;
+
+    fprintf(stdout, "%" PRIu64 "\n", hashed);
+
+    return 0;
+}