darshan-convert.c 6.86 KB
Newer Older
1
/*
Kevin Harms's avatar
Kevin Harms committed
2
 *  (C) 2011 by Argonne National Laboratory.
3 4 5 6 7 8 9 10 11 12 13 14 15 16
 *      See COPYRIGHT in top-level directory.
 */

#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <zlib.h>
#include <time.h>
#include <stdlib.h>
#include <getopt.h>
#include <assert.h>
17
#include <errno.h>
18 19 20

#include "darshan-logutils.h"

21
extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
22 23 24 25 26 27

int usage (char *exename)
{
    fprintf(stderr, "Usage: %s [options] <infile> <outfile>\n", exename);
    fprintf(stderr, "       Converts darshan log from infile to outfile.\n");
    fprintf(stderr, "       rewrites the log file into the newest format.\n");
28
    fprintf(stderr, "       --obfuscate Obfuscate items in the log.\n");
29 30
    fprintf(stderr, "       --key <key> Key to use when obfuscating.\n");
    fprintf(stderr, "       --annotate <string> Additional metadata to add.\n");
31 32 33 34

    exit(1);
}

35
void parse_args (int argc, char **argv, char **infile, char **outfile,
36
                 int *obfuscate, int *key, char **annotate)
37 38 39 40
{
    int index;
    static struct option long_opts[] =
    {
41
        {"annotate", 1, NULL, 'a'},
42 43
        {"obfuscate", 0, NULL, 'o'},
        {"key", 1, NULL, 'k'},
44 45 46 47 48 49 50 51 52 53 54
        {"help",  0, NULL, 0}
    };

    while(1)
    {
        int c = getopt_long(argc, argv, "", long_opts, &index);

        if (c == -1) break;

        switch(c)
        {
55 56 57
            case 'a':
                *annotate = optarg;
                break;
58 59 60 61 62 63
            case 'o':
                *obfuscate = 1;
                break;
            case 'k':
                *key = atoi(optarg);
                break;
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
            case 0:
            case '?':
            default:
                usage(argv[0]);
                break;
        }
    }

    if (optind + 2 == argc)
    {
        *infile = argv[optind];
        *outfile = argv[optind+1];
    }
    else
    {
        usage(argv[0]);
    }

    return;
}

85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
void obfuscate_job(int key, struct darshan_job *job)
{
    job->uid   = (int64_t) darshan_hashlittle(&job->uid, sizeof(job->uid), key);
    if (job->jobid != 0)
    {
        job->jobid = (int64_t) darshan_hashlittle(&job->jobid, sizeof(job->jobid), key);
    }

    return;
}

void obfuscate_exe(int key, char *exe)
{
    uint32_t hashed;

    hashed = darshan_hashlittle(exe, strlen(exe), key);
    memset(exe, 0, strlen(exe));
    sprintf(exe, "%u", hashed);

    return;
}

void obfuscate_file(int key, struct darshan_file *file)
{
    uint32_t hashed;

    hashed = darshan_hashlittle(file->name_suffix, sizeof(file->name_suffix), key);
    memset(file->name_suffix, 0, sizeof(file->name_suffix));
    sprintf(file->name_suffix, "%u", hashed);

    return;
}

118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
void add_annotation (char *annotation,
                     struct darshan_job *job)
{
    char *token;
    char *save;

    for(token=strtok_r(annotation, "\t", &save);
        token != NULL;
        token=strtok_r(NULL, "\t", &save))
    {
        strcat(job->metadata, token);
        strcat(job->metadata, "\n");
    }

    return;
}

135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
int main(int argc, char **argv)
{
    int ret;
    char *infile_name;
    char *outfile_name;
    struct darshan_job job;
    struct darshan_file cp_file;
    char tmp_string[1024];
    darshan_fd infile;
    darshan_fd outfile;
    int i;
    int mount_count;
    int64_t* devs;
    char** mnt_pts;
    char** fs_types;
    int last_rank = 0;
151 152
    int obfuscate = 0;
    int key = 0;
153
    char *annotation = NULL;
154

155
    parse_args(argc, argv, &infile_name, &outfile_name, &obfuscate, &key, &annotation);
156 157 158 159

    infile = darshan_log_open(infile_name, "r");
    if(!infile)
    {
160
        fprintf(stderr, "darshan_log_open() failed to open %s\n.", infile_name);
161 162 163 164 165 166 167 168 169
        return(-1);
    }
 
    /* TODO: safety check that outfile_name doesn't exist; we don't want to
     * overwrite something by accident.
     */
    outfile = darshan_log_open(outfile_name, "w");
    if(!outfile)
    {
170
        fprintf(stderr, "darshan_log_open() failed to open %s\n.", outfile_name);
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
        return(-1);
    }

    /* TODO: for now this tool is just reading the input file and throwing
     * away the data.  Need to write the log_put*() functions and use this
     * program as a test harness
     */
  
    /* read job info */
    ret = darshan_log_getjob(infile, &job);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to read job information from log file.\n");
        darshan_log_close(infile);
        return(-1);
    }

188
    if (obfuscate) obfuscate_job(key, &job);
189
    if (annotation) add_annotation(annotation, &job);
190

191 192 193 194 195 196 197 198
    ret = darshan_log_putjob(outfile, &job);
    if (ret < 0)
    {
        fprintf(stderr, "Error: unable to write job information to log file.\n");
        darshan_log_close(outfile);
        return(-1);
    }

199
    ret = darshan_log_getexe(infile, tmp_string);
200 201 202 203 204 205
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to read trailing job information.\n");
        darshan_log_close(infile);
        return(-1);
    }
206

207 208
    if (obfuscate) obfuscate_exe(key, tmp_string);

209 210 211 212 213 214 215
    ret = darshan_log_putexe(outfile, tmp_string);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to write trailing job information.\n");
        darshan_log_close(outfile);
        return(-1);
    }
216
   
217
    ret = darshan_log_getmounts(infile, &devs, &mnt_pts, &fs_types, &mount_count);
218 219 220 221 222 223 224
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to read trailing job information.\n");
        darshan_log_close(infile);
        return(-1);
    }

225 226 227 228 229 230 231 232
    ret = darshan_log_putmounts(outfile, devs, mnt_pts, fs_types, mount_count);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to write mount information.\n");
        darshan_log_close(outfile);
        return(-1);
    }

233 234 235 236 237 238 239
    ret = darshan_log_getfile(infile, &job, &cp_file);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to process log file.\n");
        fflush(stderr);
    }
    if(ret == 0)
240
    {
241
        goto done;
242 243
    }

244
    do
245 246 247 248 249 250 251 252 253
    {
        if(cp_file.rank != -1 && cp_file.rank < last_rank)
        {
            fprintf(stderr, "Error: log file contains out of order rank data.\n");
            fflush(stderr);
            return(-1);
        }
        if(cp_file.rank != -1)
            last_rank = cp_file.rank;
254

255 256
        if (obfuscate) obfuscate_file(key, &cp_file);

257 258 259 260 261 262
        ret = darshan_log_putfile(outfile, &job, &cp_file);
        if (ret < 0)
        {
            fprintf(stderr, "Error: failed to write file record.\n");
            break;
        }
263
    } while((ret = darshan_log_getfile(infile, &job, &cp_file)) == 1);
264 265 266

    if(ret < 0)
    {
267
        fprintf(stderr, "Error: failed to process log file.\n");
268 269 270
        fflush(stderr);
    }

271
done:
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
    for(i=0; i<mount_count; i++)
    {
        free(mnt_pts[i]);
        free(fs_types[i]);
    }
    if(mount_count > 0)
    {
        free(devs);
        free(mnt_pts);
        free(fs_types);
    }
 
    darshan_log_close(infile);
    darshan_log_close(outfile);

287
    return(ret);
288 289
}

290