Commit 8e765fd3 authored by Kevin Harms's avatar Kevin Harms

Perl script to iterate through a list of logs and convert them, adding annotations.

  ./darshan-convert-logs <key> <file_with_annotations> <file_with_log_names> <output directory>


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@440 3b7491f3-a168-0410-bf4b-c445ed680a29
parent eb3c1188
......@@ -56,6 +56,9 @@ darshan-analyzer: darshan-analyzer.c darshan.h darshan-log-format.h darshan-logu
darshan-log-params: darshan-log-params.c darshan-log-format.h
gcc $(CFLAGS) $(LIBS) $< -o $@
jenkins: util/bin/jenkins.o lib/lookup3.o
gcc $(CFLAGS) $(LIBS) $< -o $@ lib/lookup3.o
darshan-diff: darshan-diff.o darshan.h darshan-log-format.h darshan-logutils.o darshan-logutils.h
gcc $(CFLAGS) $(LIBS) $< darshan-logutils.o -o $@
darshan-diff.o: darshan-diff.c
......
#!/usr/bin/perl -w
#
# (C) 2011 by Argonne National Laboratory.
# See COPYRIGHT in top-level directory.
#
# Convert a list of logs and add metadata from another file.
#
# List of Logs: logfile path, one per line.
# /path/to/log1
# /path/to/log2
#
# Metadata: List of jobids with additional metadata, one per line, items are
# tab separated.
# jobid1 key1=val1 key2=val2
# jobid2 key1=val1 key2=val2
#
my $darshan_convert = "./darshan-convert";
sub load_annotations($$)
{
my $fname = shift;
my $ref = shift;
my $line;
open(FH, "<$fname") or die("Failed to open annotations: $fname\n");
while($line=<FH>)
{
my ($jobid, $annotation) = split(/\t/,$line,2);
chomp($annotation);
$ref->{$jobid} = $annotation;
}
close(FH);
}
sub main()
{
my $annotation_list;
my $logfile_list;
my $hash_key;
my $output_path;
my $logfile;
my $ref = {};
$hash_key=$ARGV[0];
$annotation_list=$ARGV[1];
$logfile_list=$ARGV[2];
$output_path=$ARGV[3];
load_annotations($annotation_list, $ref);
open(LOGFILES, "<$logfile_list") or die("Can't open: $logfile_list");
while($logfile=<LOGFILES>)
{
chomp($logfile);
if ($logfile =~ /_id(\d+)_/)
{
my $jobid;
my $annotation;
my $hashed_fname;
my @args;
$jobid = $1;
$annotation = $ref->{$jobid};
$hashed_fname = `./jenkins --64 --key $hash_key $logfile`;
chomp($hashed_fname);
@args = ("$darshan_convert",
"--obfuscate",
"--key=$hash_key",
"--annotate=$annotation",
"$logfile",
"$output_path/$hashed_fname.gz");
system(@args);
print("$hashed_fname\t$logfile\n");
}
else
{
print("Invalid logfile name: $logfile\n")
}
}
return 0;
}
#
# Main
#
&main
/*
* (C) 2011 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
/*
* Small utility to do Jenkins hashes
*/
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <inttypes.h>
#include <getopt.h>
#include <string.h>
#define TYPE_32 (1)
#define TYPE_64 (2)
extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
extern void hashlittle2(const void *key, size_t length, uint32_t *pc, uint32_t *pb);
void usage(char *progname)
{
fprintf(stderr, "usage: %s [--32] [--64] [--key <key>] data\n", progname);
exit(1);
}
void parse_args (int argc, char **argv, int* type, int* key, char **data)
{
int index;
static struct option long_opts[] =
{
{"32", 0, NULL, 'a'},
{"64", 0, NULL, 'b'},
{"key", 1, NULL, 'k'},
{"help", 0, NULL, 0}
};
while(1)
{
int c = getopt_long(argc, argv, "", long_opts, &index);
if (c == -1) break;
switch(c)
{
case 'a':
*type = TYPE_32;
break;
case 'b':
*type = TYPE_64;
break;
case 'k':
*key = atoi(optarg);
break;
case 0:
case '?':
default:
usage(argv[0]);
break;
}
}
if (optind + 1 == argc)
{
*data = argv[optind];
}
else
{
usage(argv[0]);
}
return;
}
int main (int argc, char **argv)
{
int type = TYPE_32;
int key = 0;
char *data;
uint32_t hashed_u = 0;
uint32_t hashed_l = 0;
uint64_t hashed;
parse_args (argc, argv, &type, &key, &data);
switch(type)
{
case TYPE_64:
hashed_u = key;
hashlittle2(data, strlen(data), &hashed_u, &hashed_l);
break;
case TYPE_32:
default:
hashed_l = darshan_hashlittle(data, strlen(data), key);
break;
}
hashed = ((uint64_t)hashed_u << 32) | hashed_l;
fprintf(stdout, "%" PRIu64 "\n", hashed);
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment