darshan-convert.c 7.3 KB
Newer Older
1
/*
Kevin Harms's avatar
Kevin Harms committed
2
 *  (C) 2011 by Argonne National Laboratory.
3
4
5
6
7
8
9
10
11
12
13
14
15
16
 *      See COPYRIGHT in top-level directory.
 */

#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <zlib.h>
#include <time.h>
#include <stdlib.h>
#include <getopt.h>
#include <assert.h>
17
#include <errno.h>
18
19
20

#include "darshan-logutils.h"

21
extern uint32_t darshan_hashlittle(const void *key, size_t length, uint32_t initval);
22
23
24
25
26
27

int usage (char *exename)
{
    fprintf(stderr, "Usage: %s [options] <infile> <outfile>\n", exename);
    fprintf(stderr, "       Converts darshan log from infile to outfile.\n");
    fprintf(stderr, "       rewrites the log file into the newest format.\n");
28
    fprintf(stderr, "       --obfuscate Obfuscate items in the log.\n");
29
30
    fprintf(stderr, "       --key <key> Key to use when obfuscating.\n");
    fprintf(stderr, "       --annotate <string> Additional metadata to add.\n");
31
    fprintf(stderr, "       --file <hash> Limit output to specified (hashed) file only.\n");
32
33
34
35

    exit(1);
}

36
void parse_args (int argc, char **argv, char **infile, char **outfile,
37
                 int *obfuscate, int *key, char **annotate, uint64_t* hash)
38
39
{
    int index;
40
41
    int ret;

42
43
    static struct option long_opts[] =
    {
44
        {"annotate", 1, NULL, 'a'},
45
46
        {"obfuscate", 0, NULL, 'o'},
        {"key", 1, NULL, 'k'},
47
        {"file", 1, NULL, 'f'},
48
49
50
        {"help",  0, NULL, 0}
    };

51
52
    *hash = 0;

53
54
55
56
57
58
59
60
    while(1)
    {
        int c = getopt_long(argc, argv, "", long_opts, &index);

        if (c == -1) break;

        switch(c)
        {
61
62
63
            case 'a':
                *annotate = optarg;
                break;
64
65
66
67
68
69
            case 'o':
                *obfuscate = 1;
                break;
            case 'k':
                *key = atoi(optarg);
                break;
70
71
72
73
74
            case 'f':
                ret = sscanf(optarg, "%" PRIu64, hash);
                if(ret != 1)
                    usage(argv[0]);
                break;
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
            case 0:
            case '?':
            default:
                usage(argv[0]);
                break;
        }
    }

    if (optind + 2 == argc)
    {
        *infile = argv[optind];
        *outfile = argv[optind+1];
    }
    else
    {
        usage(argv[0]);
    }

    return;
}

96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
void obfuscate_job(int key, struct darshan_job *job)
{
    job->uid   = (int64_t) darshan_hashlittle(&job->uid, sizeof(job->uid), key);
    if (job->jobid != 0)
    {
        job->jobid = (int64_t) darshan_hashlittle(&job->jobid, sizeof(job->jobid), key);
    }

    return;
}

void obfuscate_exe(int key, char *exe)
{
    uint32_t hashed;

    hashed = darshan_hashlittle(exe, strlen(exe), key);
    memset(exe, 0, strlen(exe));
    sprintf(exe, "%u", hashed);

    return;
}

void obfuscate_file(int key, struct darshan_file *file)
{
    uint32_t hashed;

    hashed = darshan_hashlittle(file->name_suffix, sizeof(file->name_suffix), key);
    memset(file->name_suffix, 0, sizeof(file->name_suffix));
    sprintf(file->name_suffix, "%u", hashed);

    return;
}

129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
void add_annotation (char *annotation,
                     struct darshan_job *job)
{
    char *token;
    char *save;

    for(token=strtok_r(annotation, "\t", &save);
        token != NULL;
        token=strtok_r(NULL, "\t", &save))
    {
        strcat(job->metadata, token);
        strcat(job->metadata, "\n");
    }

    return;
}

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
int main(int argc, char **argv)
{
    int ret;
    char *infile_name;
    char *outfile_name;
    struct darshan_job job;
    struct darshan_file cp_file;
    char tmp_string[1024];
    darshan_fd infile;
    darshan_fd outfile;
    int i;
    int mount_count;
    int64_t* devs;
    char** mnt_pts;
    char** fs_types;
    int last_rank = 0;
162
163
    int obfuscate = 0;
    int key = 0;
164
    char *annotation = NULL;
165
    uint64_t hash;
166

167
    parse_args(argc, argv, &infile_name, &outfile_name, &obfuscate, &key, &annotation, &hash);
168
169
170
171

    infile = darshan_log_open(infile_name, "r");
    if(!infile)
    {
172
        fprintf(stderr, "darshan_log_open() failed to open %s\n.", infile_name);
173
174
175
176
177
178
179
180
181
        return(-1);
    }
 
    /* TODO: safety check that outfile_name doesn't exist; we don't want to
     * overwrite something by accident.
     */
    outfile = darshan_log_open(outfile_name, "w");
    if(!outfile)
    {
182
        fprintf(stderr, "darshan_log_open() failed to open %s\n.", outfile_name);
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
        return(-1);
    }

    /* TODO: for now this tool is just reading the input file and throwing
     * away the data.  Need to write the log_put*() functions and use this
     * program as a test harness
     */
  
    /* read job info */
    ret = darshan_log_getjob(infile, &job);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to read job information from log file.\n");
        darshan_log_close(infile);
        return(-1);
    }

200
    if (obfuscate) obfuscate_job(key, &job);
201
    if (annotation) add_annotation(annotation, &job);
202

203
204
205
206
207
208
209
210
    ret = darshan_log_putjob(outfile, &job);
    if (ret < 0)
    {
        fprintf(stderr, "Error: unable to write job information to log file.\n");
        darshan_log_close(outfile);
        return(-1);
    }

211
    ret = darshan_log_getexe(infile, tmp_string);
212
213
214
215
216
217
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to read trailing job information.\n");
        darshan_log_close(infile);
        return(-1);
    }
218

219
220
    if (obfuscate) obfuscate_exe(key, tmp_string);

221
222
223
224
225
226
227
    ret = darshan_log_putexe(outfile, tmp_string);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to write trailing job information.\n");
        darshan_log_close(outfile);
        return(-1);
    }
228
   
229
    ret = darshan_log_getmounts(infile, &devs, &mnt_pts, &fs_types, &mount_count);
230
231
232
233
234
235
236
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to read trailing job information.\n");
        darshan_log_close(infile);
        return(-1);
    }

237
238
239
240
241
242
243
244
    ret = darshan_log_putmounts(outfile, devs, mnt_pts, fs_types, mount_count);
    if(ret < 0)
    {
        fprintf(stderr, "Error: unable to write mount information.\n");
        darshan_log_close(outfile);
        return(-1);
    }

245
246
247
248
249
250
251
    ret = darshan_log_getfile(infile, &job, &cp_file);
    if(ret < 0)
    {
        fprintf(stderr, "Error: failed to process log file.\n");
        fflush(stderr);
    }
    if(ret == 0)
252
    {
253
        goto done;
254
255
    }

256
    do
257
258
259
260
261
262
263
264
265
    {
        if(cp_file.rank != -1 && cp_file.rank < last_rank)
        {
            fprintf(stderr, "Error: log file contains out of order rank data.\n");
            fflush(stderr);
            return(-1);
        }
        if(cp_file.rank != -1)
            last_rank = cp_file.rank;
266

267
        if(!hash || hash == cp_file.hash)
268
        {
269
270
271
272
273
274
275
276
            if (obfuscate) obfuscate_file(key, &cp_file);

            ret = darshan_log_putfile(outfile, &job, &cp_file);
            if (ret < 0)
            {
                fprintf(stderr, "Error: failed to write file record.\n");
                break;
            }
277
        }
278
    } while((ret = darshan_log_getfile(infile, &job, &cp_file)) == 1);
279
280
281

    if(ret < 0)
    {
282
        fprintf(stderr, "Error: failed to process log file.\n");
283
284
285
        fflush(stderr);
    }

286
done:
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
    for(i=0; i<mount_count; i++)
    {
        free(mnt_pts[i]);
        free(fs_types[i]);
    }
    if(mount_count > 0)
    {
        free(devs);
        free(mnt_pts);
        free(fs_types);
    }
 
    darshan_log_close(infile);
    darshan_log_close(outfile);

302
    return(ret);
303
304
}

305