Commit 03a2ec99 authored by Philip Carns's avatar Philip Carns

added some utilities for running fsstats in parallel using ssh


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@164 3b7491f3-a168-0410-bf4b-c445ed680a29
parent 306ec134
#!/bin/bash
#
# (C) 2010 by Argonne National Laboratory.
# See COPYRIGHT in top-level directory.
#
# usage: fsstats-runner.bash <hostname> <path>
# make this configurable or something...
FSSTATS_PATH=/home/pcarns/working/parallel-fsstats
if [ "${#}" != 2 ]
then
echo "Error: bad arguments"
exit 1
fi
HOST=${1}
FS_PATH=${2}
# the output file name will be the path that was scanned, but with equals
# signs in place of the slashes
OUT_FILE=`echo $FS_PATH | sed -e 's/\//=/g'`
OUT_FILE="${OUT_FILE}.csv"
# launch remote command
ssh -oBatchMode=yes -n $HOST "${FSSTATS_PATH}/fsstats -o /tmp/pfsstats-$$.csv $FS_PATH >& /dev/null"
if [ "${?}" != 0 ]
then
exit 1
fi
# retrieve output file
scp -oBatchMode=yes $HOST:/tmp/pfsstats-$$.csv $OUT_FILE >& /dev/null
if [ "${?}" != 0 ]
then
exit 1
fi
# delete file on remote host
ssh -oBatchMode=yes -n $HOST "rm -f /tmp/pfsstats-$$.csv >& /dev/null"
if [ "${?}" != 0 ]
then
exit 1
fi
exit 0
#!/usr/bin/perl -w
#
# (C) 2010 by Argonne National Laboratory.
# See COPYRIGHT in top-level directory.
#
use Cwd;
use Getopt::Long;
use English;
my $hostfile;
my $dirfile;
my @hosts;
my @dirs;
my %pidmapping = ();
my %pidmapping_dir = ();
process_args();
open(FILE, $hostfile) or die("Error opening hostfile.");
# read file into an array
@hosts = <FILE>;
chomp(@hosts);
close(FILE);
open(FILE, $dirfile) or die("Error opening dirfile.");
# read file into an array
@dirs = <FILE>;
chomp(@dirs);
close(FILE);
# loop as long as we still have directories to process or outstanding jobs
while ($#dirs > -1 || keys(%pidmapping))
{
if($#hosts > -1 && $#dirs > -1)
{
# we have work to do and a free host to do it on
# grab a directory and host out of the lists
my $dir = pop(@dirs);
my $host = pop(@hosts);
my $pid = fork();
if (not defined $pid)
{
die("Error: could not fork.");
}
my $cmd = "./fsstats-runner.bash $host $dir";
if($pid == 0)
{
# child runs remote command
my $error_code = 0;
system($cmd);
# look for exec problem, signal, or command error code
if (($? == -1) || ($? & 127) || (($? >> 8) != 0))
{
$error_code = 1;
}
else
{
$error_code = 0;
}
# propagate an error code to parent
exit($error_code);
}
else
{
print("fsstats of $dir on $host started...\n");
# parent keeps up with what child is running where
$pidmapping{$pid} = $host;
$pidmapping_dir{$pid} = $dir;
}
}
else
{
# we have launched as much as we can; wait for something to finish
my $child = waitpid(-1, 0);
if($child < 1)
{
die("Error: lost track of a child process.\n");
}
my $child_error_code = $?;
print("fsstats of $pidmapping_dir{$child} on $pidmapping{$child} ");
if($child_error_code == 0)
{
print(" SUCCESS.\n");
}
else
{
print(" FAILURE (continuing).\n");
}
# put the host back on the queue
push(@hosts, $pidmapping{$child});
delete($pidmapping{$child});
delete($pidmapping_dir{$child});
}
}
sub process_args
{
use vars qw( $opt_help );
Getopt::Long::Configure("no_ignore_case", "bundling");
GetOptions( "help" );
if($opt_help)
{
print_help();
exit(0);
}
# there should be two remaining arguments (hostfile and dirfile)
if($#ARGV != 1)
{
print "Error: invalid arguments.\n";
print_help();
exit(1);
}
$hostfile = $ARGV[0];
$dirfile = $ARGV[1];
return;
}
sub print_help
{
print <<EOF;
Usage: $PROGRAM_NAME <hostfile> <dirfile>
--help Prints this help message
Purpose:
This script runs parallel copies of fsstats on each directory listed in
the <dirfile>. The <hostfile> specifies a list of hosts to run fsstats
on via ssh.
EOF
return;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment