darshan-apmpi-logutils.c 21.7 KB
Newer Older
Sudheer Chunduri's avatar
Sudheer Chunduri committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (C) 2018 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _GNU_SOURCE
#include "darshan-util-config.h"
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <unistd.h>
#include <inttypes.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>

#include "darshan-logutils.h"
#include "darshan-apmpi-log-format.h"

Sudheer Chunduri's avatar
Sudheer Chunduri committed
23
/* counter name strings for the MPI module */
Sudheer Chunduri's avatar
Sudheer Chunduri committed
24 25 26
#define Y(a) #a, 
#define Z(a) #a
#define X I
27
#define V J
Sudheer Chunduri's avatar
Sudheer Chunduri committed
28
char *apmpi_counter_names[] = {
29
    APMPI_MPIOP_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
30
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
31
#undef X
32
#undef V
33 34
#define X F_TIME
#define V F_TIME
35 36 37 38
char *apmpi_f_mpiop_totaltime_counter_names[] = {
    APMPI_F_MPIOP_TOTALTIME_COUNTERS
};
#undef X
39
#undef V
40
#define X F_SYNC
41
#define V F_SYNC
42
char *apmpi_f_mpiop_synctime_counter_names[] = {
43 44 45
    APMPI_F_MPIOP_SYNCTIME_COUNTERS
};
#undef X
46
#undef V
47 48
char *apmpi_f_mpi_global_counter_names[] = {
    APMPI_F_MPI_GLOBAL_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
49
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
50 51 52
#undef Y
#undef Z

Sudheer Chunduri's avatar
Sudheer Chunduri committed
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p);
static int darshan_log_put_apmpi_rec(darshan_fd fd, void* buf);
static void darshan_log_print_apmpi_rec(void *file_rec,
    char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_print_apmpi_description(int ver);
static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
    void *file_rec2, char *file_name2);

struct darshan_mod_logutil_funcs apmpi_logutils =
{
    .log_get_record = &darshan_log_get_apmpi_rec,
    .log_put_record = &darshan_log_put_apmpi_rec,
    .log_print_record = &darshan_log_print_apmpi_rec,
    .log_print_description = &darshan_log_print_apmpi_description,
    .log_print_diff = &darshan_log_print_apmpi_rec_diff,
    .log_agg_records = NULL
};

71

Sudheer Chunduri's avatar
Sudheer Chunduri committed
72 73
static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p)
{
74
    struct darshan_apmpi_header_record *hdr_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
75 76 77 78 79
    struct darshan_apmpi_perf_record *prf_rec;
    int rec_len;
    char *buffer;
    int i;
    int ret = -1;
80
    static int first_rec = 1;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
81

82
    if(fd->mod_map[DARSHAN_APMPI_MOD].len == 0)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
83 84
        return(0);

85 86 87 88 89 90 91 92
    if(fd->mod_ver[DARSHAN_APMPI_MOD] == 0 ||
        fd->mod_ver[DARSHAN_APMPI_MOD] > APMPI_VER)
    {
        fprintf(stderr, "Error: Invalid APMPI module version number (got %d)\n",
            fd->mod_ver[DARSHAN_APMPI_MOD]);
        return(-1);
    }

Sudheer Chunduri's avatar
Sudheer Chunduri committed
93 94 95 96 97 98 99 100 101 102 103 104 105 106
    if (!*buf_p)
    {
        /* assume this is the largest possible record size */
        buffer = malloc(sizeof(struct darshan_apmpi_perf_record));
        if (!buffer)
        {
            return(-1);
        }
    }
    else
    {
        buffer = *buf_p;
    }

107
    if (fd->mod_ver[DARSHAN_APMPI_MOD] < APMPI_VER)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
108 109 110 111 112
    {
        /* perform conversion as needed */
    }

    /* v1, current version */
113
    if (fd->mod_ver[DARSHAN_APMPI_MOD] == APMPI_VER)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
114
    {
115 116 117 118 119 120
        if (first_rec)
        {
            rec_len = sizeof(struct darshan_apmpi_header_record);
            first_rec = 0;
        }
        else
Sudheer Chunduri's avatar
Sudheer Chunduri committed
121 122
            rec_len = sizeof(struct darshan_apmpi_perf_record);

123
        ret = darshan_log_get_mod(fd, DARSHAN_APMPI_MOD, buffer, rec_len);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
124 125 126 127 128 129
    }

    if (ret == rec_len)
    {
        if(fd->swap_flag)
        {
130 131 132 133 134 135
            if (rec_len == sizeof(struct darshan_apmpi_header_record))
            {
                hdr_rec = (struct darshan_apmpi_header_record*)buffer;
                /* swap bytes if necessary */
                DARSHAN_BSWAP64(&(hdr_rec->base_rec.id));
                DARSHAN_BSWAP64(&(hdr_rec->base_rec.rank));
Shane Snyder's avatar
Shane Snyder committed
136 137
                DARSHAN_BSWAP64(&(hdr_rec->magic));
                DARSHAN_BSWAP32(&(hdr_rec->sync_flag));
138
                DARSHAN_BSWAP64(&(hdr_rec->apmpi_f_variance_total_mpitime));
139
                DARSHAN_BSWAP64(&(hdr_rec->apmpi_f_variance_total_mpisynctime));
140 141 142
            }
            else
            {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
143 144 145 146 147 148 149
                prf_rec = (struct darshan_apmpi_perf_record*)buffer;
                DARSHAN_BSWAP64(&(prf_rec->base_rec.id));
                DARSHAN_BSWAP64(&(prf_rec->base_rec.rank));
                for (i = 0; i < APMPI_NUM_INDICES; i++)
                {
                    DARSHAN_BSWAP64(&prf_rec->counters[i]);
                }
150
                for (i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
151 152 153
                {
                    DARSHAN_BSWAP64(&prf_rec->fcounters[i]);
                }
154
                for (i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
155 156 157
                {
                    DARSHAN_BSWAP64(&prf_rec->fsynccounters[i]);
                }
158
                for (i = 0; i < APMPI_F_MPI_GLOBAL_NUM_INDICES; i++)
159 160 161 162
                {
                    DARSHAN_BSWAP64(&prf_rec->fglobalcounters[i]);
                }
            }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
163 164 165 166 167 168
        }
        *buf_p = buffer;
        return(1);
    }
    else if (ret < 0)
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
169
 //       *buf_p = NULL;
170
        if (!*buf_p) free(buffer);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
171 172 173 174
        return(-1);
    }
    else
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
175
  //      *buf_p = NULL;
176
        if (!*buf_p) free(buffer);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
177 178 179 180 181 182 183 184 185 186
        return(0);
    }
}

static int darshan_log_put_apmpi_rec(darshan_fd fd, void* buf)
{
    int ret;
    int rec_len;
    static int first_rec = 1;

187 188 189 190 191 192 193
    if (first_rec)
    {
        rec_len = sizeof(struct darshan_apmpi_header_record);
        first_rec = 0;
    }
    else
        rec_len = sizeof(struct darshan_apmpi_perf_record);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
194
    
195
    ret = darshan_log_put_mod(fd, DARSHAN_APMPI_MOD, buf,
Sudheer Chunduri's avatar
Sudheer Chunduri committed
196
                              rec_len, APMPI_VER);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
197 198 199 200 201 202 203 204 205 206
    if(ret < 0)
        return(-1);

    return(0);
}

static void darshan_log_print_apmpi_rec(void *rec, char *file_name,
    char *mnt_pt, char *fs_type)
{
    int i;
207
    static int first_rec = 1;
208
    static int sync_flag;
209
    struct darshan_apmpi_header_record *hdr_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
210
    struct darshan_apmpi_perf_record *prf_rec;
211 212 213 214
    
    if (first_rec)
    {
        hdr_rec = rec;
215
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
216
            hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
217
            "MPI_TOTAL_COMM_TIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpitime,
218
            "", "", "");
219
        if(hdr_rec->sync_flag)
220
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
221
            hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
222
            "MPI_TOTAL_COMM_SYNC_TIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpisynctime,
223
            "", "", "");
224
        first_rec = 0;
225
        sync_flag = hdr_rec->sync_flag;
226 227 228
    }
    else
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
229
        prf_rec = rec;
230

231
        DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
232
                  prf_rec->base_rec.rank, prf_rec->base_rec.id,
233
                  "MPI_PROCESSOR_NAME", prf_rec->node_name,
234 235
                  "", "", "");
   
Sudheer Chunduri's avatar
Sudheer Chunduri committed
236 237
        for(i = 0; i < APMPI_NUM_INDICES; i++)
        {
238
            DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
239 240 241 242
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
                apmpi_counter_names[i], prf_rec->counters[i],
                "", "", "");
        }
243
        for(i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
244
        {
245
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
246
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
247 248 249
                apmpi_f_mpiop_totaltime_counter_names[i], prf_rec->fcounters[i],
                "", "", "");
        }
250
        if(sync_flag)
251
        {
252 253 254 255 256 257 258
            for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
            {
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
                    prf_rec->base_rec.rank, prf_rec->base_rec.id,
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec->fsynccounters[i],
                    "", "", "");
            }
259
        }
260 261 262
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
            prf_rec->base_rec.rank, prf_rec->base_rec.id,
            apmpi_f_mpi_global_counter_names[0], prf_rec->fglobalcounters[0],
263
                "", "", "");
264 265
        if(sync_flag)
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
266 267
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
                apmpi_f_mpi_global_counter_names[1], prf_rec->fglobalcounters[1],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
268
                "", "", "");
269
    }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
270 271 272 273 274 275

    return;
}

static void darshan_log_print_apmpi_description(int ver)
{
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
    printf("\n# description of APMPI counters:\n");
    printf("#   global summary stats showing the variance across all MPI processes:\n");
    printf("#     MPI_TOTAL_COMM_TIME_VARIANCE: variance in total communication time across all the processes.\n");
    printf("#     MPI_TOTAL_COMM_SYNC_TIME_VARIANCE: variance in total sync time across all the processes, if enabled.\n");
    printf("#   per-process detailed stats on the usage of various MPI routines:\n");
    printf("#     MPI_PROCESSOR_NAME: name of the processor used by the MPI process.\n");
    printf("#     MPI_[OP]_*: set of statistics for each instrumented MPI op (e.g., SEND, ALLREDUCE, BCAST, etc.).\n");
    printf("#       CALL_COUNT: total call count for an MPI op.\n");
    printf("#       TOTAL_BYTES: total bytes (i.e., cumulative across all calls) moved with an MPI op.\n");
    printf("#       MSG_SIZE_AGG_*: histogram of total bytes moved for all the calls of an MPI op.\n");
    printf("#       TOTAL_TIME: total time (i.e, cumulative across all calls) of an MPI op.\n");
    printf("#       MIN_TIME: minimum time across all calls of an MPI op.\n");
    printf("#       MAX_TIME: maximum time across all calls of an MPI op.\n");
    printf("#       TOTAL_SYNC_TIME: total sync time (cumulative across all calls of an op) of an MPI op, if enabled.\n");
    printf("#     MPI_TOTAL_COMM_TIME: total communication (MPI) time of a process across all the MPI ops.\n");
    printf("#     MPI_TOTAL_COMM_SYNC_TIME: total sync time of a process across all the MPI ops, if enabled.\n");
Sudheer Chunduri's avatar
Sudheer Chunduri committed
292 293 294 295 296 297
    return;
}

static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
    void *file_rec2, char *file_name2)
{
298 299
    struct darshan_apmpi_header_record *hdr_rec1;
    struct darshan_apmpi_header_record *hdr_rec2;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
300 301 302
    struct darshan_apmpi_perf_record   *prf_rec1;
    struct darshan_apmpi_perf_record   *prf_rec2;

303 304 305
    hdr_rec1 = (struct darshan_apmpi_header_record*) file_rec1;
    hdr_rec2 = (struct darshan_apmpi_header_record*) file_rec2;
    prf_rec1 = (struct darshan_apmpi_perf_record*) file_rec1;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
306
    prf_rec2 = (struct darshan_apmpi_perf_record*) file_rec2;
307 308 309
    static int sync_flag;
    sync_flag = hdr_rec1->sync_flag && hdr_rec2->sync_flag;

310 311 312 313 314 315
    if (hdr_rec1->magic == APMPI_MAGIC)
    {
        /* this is the header record */   
        if (!hdr_rec2) 
        {
            printf("- ");   
316
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
317
                hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
318
                "MPI_TOTAL_COMM_TIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
319
                "", "", "");
320
            if(sync_flag)
321
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
322
                hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
323
                "MPI_TOTAL_COMM_SYNC_TIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
324
                "", "", "");
325 326 327 328
        }
        else if (!hdr_rec1)
        {
            printf("+ ");
329
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
330
                hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
331
                "MPI_TOTAL_COMM_TIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
332
                "", "", "");
333
            if(sync_flag)
334
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
335
                hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
336
                "MPI_TOTAL_COMM_SYNC_TIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
337
                "", "", "");
338 339 340 341 342 343
        }
        else
        {   
            if (hdr_rec1->apmpi_f_variance_total_mpitime != hdr_rec2->apmpi_f_variance_total_mpitime)
            {
                printf("- ");
344
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
345
                    hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
346
                    "MPI_TOTAL_COMM_TIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
347 348
                    "", "", "");
                printf("+ ");
349
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
350
                    hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
351
                    "MPI_TOTAL_COMM_TIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
352 353
                    "", "", "");
            }
354 355
            if(sync_flag)
            {
356 357 358
            if (hdr_rec1->apmpi_f_variance_total_mpisynctime != hdr_rec2->apmpi_f_variance_total_mpisynctime)
            {
                printf("- ");
359
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
360
                    hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
361
                    "MPI_TOTAL_COMM_SYNC_TIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
362 363
                    "", "", "");
                printf("+ ");
364
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
365
                    hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
366
                    "MPI_TOTAL_COMM_SYNC_TIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
367 368
                    "", "", "");
            }
369
            }
370 371 372 373
        }
    }
    else
    {   
374 375 376
        if(!prf_rec2)
        {
            printf("- ");
377
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
378
                  prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
379
                  "MPI_PROCESSOR_NAME", prf_rec1->node_name,
380 381 382 383 384
                  "", "", "");
        }
        else if (!prf_rec1)
        {
            printf("+ ");
385
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
386
                  prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
387
                  "MPI_PROCESSOR_NAME", prf_rec2->node_name,
388 389
                  "", "", "");
        }
390
        else if (prf_rec1->node_name != prf_rec2->node_name)
391 392
        {
            printf("- ");
393
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
394
                  prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
395
                  "MPI_PROCESSOR_NAME", prf_rec1->node_name,
396 397
                  "", "", "");
            printf("+ ");
398
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
399
                  prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
400
                  "MPI_PROCESSOR_NAME", prf_rec2->node_name,
401 402
                  "", "", "");
        }
403 404 405 406 407 408
        int i;
        for(i = 0; i < APMPI_NUM_INDICES; i++)
        {
            if (!prf_rec2)
            {   
                printf("- ");
409
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
410 411 412 413 414 415 416
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_counter_names[i], prf_rec1->counters[i],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
417
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
418 419 420 421 422 423 424
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_counter_names[i], prf_rec2->counters[i],
                    "", "", "");
            }
            else if (prf_rec1->counters[i] != prf_rec2->counters[i])
            {
                printf("- ");
425
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
426 427 428 429
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_counter_names[i], prf_rec1->counters[i],
                    "", "", "");
                printf("+ ");
430
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
431 432 433 434 435 436
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_counter_names[i], prf_rec2->counters[i],
                    "", "", "");
            }
        }
        
437
        for(i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
438 439 440 441
        {   
            if (!prf_rec2)
            {   
                printf("- ");
442
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
443 444 445 446 447 448 449
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
450
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
451 452 453 454 455 456 457
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
                    "", "", "");
            }
            else if (prf_rec1->counters[i] != prf_rec2->counters[i])
            {
                printf("- ");
458
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
459 460 461 462
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
                    "", "", "");
                printf("+ ");
463
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
464 465 466 467 468
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
                    "", "", "");
            }
        }
469 470
        if(sync_flag)
        {
471
        for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
472 473 474 475
        {
            if (!prf_rec2)
            {
                printf("- ");
476
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
477
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
478
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
479 480 481 482 483 484
                    "", "", "");

            }
            else if (!prf_rec1)
            {
                printf("+ ");
485
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
486
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
487
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
488 489 490 491 492
                    "", "", "");
            }
            else if (prf_rec1->fsynccounters[i] != prf_rec2->fsynccounters[i])
            {
                printf("- ");
493
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
494
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
495
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
496 497
                    "", "", "");
                printf("+ ");
498
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
499
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
500
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
501 502 503
                    "", "", "");
            }
        }
504 505
        }
        //for(i = 0; i < APMPI_F_MPI_GLOBAL_NUM_INDICES; i++)
506 507 508 509
        {
            if (!prf_rec2)
            {
                printf("- ");
510
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
511
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
512
                    apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
513 514 515 516 517
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
518
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
519
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
520
                    apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
521 522
                    "", "", "");
            }
523
            else if (prf_rec1->fglobalcounters[0] != prf_rec2->fglobalcounters[0])
524 525
            {
                printf("- ");
526
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
527
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
528
                    apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
529 530
                    "", "", "");
                printf("+ ");
531
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
532
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
533
                    apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
534 535
                    "", "", "");
            }
536 537 538 539 540
            if(sync_flag)
            {
            if (!prf_rec2)
            {
                printf("- ");
541
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
542 543 544 545 546 547 548
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec1->fglobalcounters[1],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
549
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
550 551 552 553 554 555 556
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec2->fglobalcounters[1],
                    "", "", "");
            }
            else if (prf_rec1->fglobalcounters[1] != prf_rec2->fglobalcounters[1])
            {
                printf("- ");
557
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
558 559 560 561
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec1->fglobalcounters[1],
                    "", "", "");
                printf("+ ");
562
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
563 564 565 566 567 568
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec2->fglobalcounters[1],
                    "", "", "");
            }
           }
       }
569
    }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
570 571 572 573 574 575 576 577 578 579 580 581


    return;
}
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */