darshan-apmpi-logutils.c 20.3 KB
Newer Older
Sudheer Chunduri's avatar
Sudheer Chunduri committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (C) 2018 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _GNU_SOURCE
#include "darshan-util-config.h"
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <unistd.h>
#include <inttypes.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>

#include "darshan-logutils.h"
#include "darshan-apmpi-log-format.h"

Sudheer Chunduri's avatar
Sudheer Chunduri committed
23
/* counter name strings for the MPI module */
Sudheer Chunduri's avatar
Sudheer Chunduri committed
24 25 26
#define Y(a) #a, 
#define Z(a) #a
#define X I
27
#define V J
Sudheer Chunduri's avatar
Sudheer Chunduri committed
28
char *apmpi_counter_names[] = {
29
    APMPI_MPIOP_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
30
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
31
#undef X
32
#undef V
33 34
#define X F_TIME
#define V F_TIME
35 36 37 38
char *apmpi_f_mpiop_totaltime_counter_names[] = {
    APMPI_F_MPIOP_TOTALTIME_COUNTERS
};
#undef X
39
#undef V
40
#define X F_SYNC
41
#define V F_SYNC
42
char *apmpi_f_mpiop_synctime_counter_names[] = {
43 44 45
    APMPI_F_MPIOP_SYNCTIME_COUNTERS
};
#undef X
46
#undef V
47 48
char *apmpi_f_mpi_global_counter_names[] = {
    APMPI_F_MPI_GLOBAL_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
49
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
50 51 52
#undef Y
#undef Z

Sudheer Chunduri's avatar
Sudheer Chunduri committed
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p);
static int darshan_log_put_apmpi_rec(darshan_fd fd, void* buf);
static void darshan_log_print_apmpi_rec(void *file_rec,
    char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_print_apmpi_description(int ver);
static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
    void *file_rec2, char *file_name2);

struct darshan_mod_logutil_funcs apmpi_logutils =
{
    .log_get_record = &darshan_log_get_apmpi_rec,
    .log_put_record = &darshan_log_put_apmpi_rec,
    .log_print_record = &darshan_log_print_apmpi_rec,
    .log_print_description = &darshan_log_print_apmpi_description,
    .log_print_diff = &darshan_log_print_apmpi_rec_diff,
    .log_agg_records = NULL
};

71

Sudheer Chunduri's avatar
Sudheer Chunduri committed
72 73
static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p)
{
74
    struct darshan_apmpi_header_record *hdr_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
75 76 77 78 79
    struct darshan_apmpi_perf_record *prf_rec;
    int rec_len;
    char *buffer;
    int i;
    int ret = -1;
80
    static int first_rec = 1;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
81

82
    if(fd->mod_map[DARSHAN_APMPI_MOD].len == 0)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
83 84
        return(0);

85 86 87 88 89 90 91 92
    if(fd->mod_ver[DARSHAN_APMPI_MOD] == 0 ||
        fd->mod_ver[DARSHAN_APMPI_MOD] > APMPI_VER)
    {
        fprintf(stderr, "Error: Invalid APMPI module version number (got %d)\n",
            fd->mod_ver[DARSHAN_APMPI_MOD]);
        return(-1);
    }

Sudheer Chunduri's avatar
Sudheer Chunduri committed
93 94 95 96 97 98 99 100 101 102 103 104 105 106
    if (!*buf_p)
    {
        /* assume this is the largest possible record size */
        buffer = malloc(sizeof(struct darshan_apmpi_perf_record));
        if (!buffer)
        {
            return(-1);
        }
    }
    else
    {
        buffer = *buf_p;
    }

107
    if (fd->mod_ver[DARSHAN_APMPI_MOD] < APMPI_VER)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
108 109 110 111 112
    {
        /* perform conversion as needed */
    }

    /* v1, current version */
113
    if (fd->mod_ver[DARSHAN_APMPI_MOD] == APMPI_VER)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
114
    {
115 116 117 118 119 120
        if (first_rec)
        {
            rec_len = sizeof(struct darshan_apmpi_header_record);
            first_rec = 0;
        }
        else
Sudheer Chunduri's avatar
Sudheer Chunduri committed
121 122
            rec_len = sizeof(struct darshan_apmpi_perf_record);

123
        ret = darshan_log_get_mod(fd, DARSHAN_APMPI_MOD, buffer, rec_len);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
124 125 126 127 128 129
    }

    if (ret == rec_len)
    {
        if(fd->swap_flag)
        {
130 131 132 133 134 135 136
            if (rec_len == sizeof(struct darshan_apmpi_header_record))
            {
                hdr_rec = (struct darshan_apmpi_header_record*)buffer;
                /* swap bytes if necessary */
                DARSHAN_BSWAP64(&(hdr_rec->base_rec.id));
                DARSHAN_BSWAP64(&(hdr_rec->base_rec.rank));
                DARSHAN_BSWAP64(&(hdr_rec->apmpi_f_variance_total_mpitime));
137
                DARSHAN_BSWAP64(&(hdr_rec->apmpi_f_variance_total_mpisynctime));
138 139 140
            }
            else
            {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
141 142 143 144 145 146 147
                prf_rec = (struct darshan_apmpi_perf_record*)buffer;
                DARSHAN_BSWAP64(&(prf_rec->base_rec.id));
                DARSHAN_BSWAP64(&(prf_rec->base_rec.rank));
                for (i = 0; i < APMPI_NUM_INDICES; i++)
                {
                    DARSHAN_BSWAP64(&prf_rec->counters[i]);
                }
148
                for (i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
149 150 151
                {
                    DARSHAN_BSWAP64(&prf_rec->fcounters[i]);
                }
152
                for (i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
153 154 155
                {
                    DARSHAN_BSWAP64(&prf_rec->fsynccounters[i]);
                }
156
                for (i = 0; i < APMPI_F_MPI_GLOBAL_NUM_INDICES; i++)
157 158 159 160
                {
                    DARSHAN_BSWAP64(&prf_rec->fglobalcounters[i]);
                }
            }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
161 162 163 164 165 166
        }
        *buf_p = buffer;
        return(1);
    }
    else if (ret < 0)
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
167
 //       *buf_p = NULL;
168
        if (!*buf_p) free(buffer);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
169 170 171 172
        return(-1);
    }
    else
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
173
  //      *buf_p = NULL;
174
        if (!*buf_p) free(buffer);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
175 176 177 178 179 180 181 182 183 184
        return(0);
    }
}

static int darshan_log_put_apmpi_rec(darshan_fd fd, void* buf)
{
    int ret;
    int rec_len;
    static int first_rec = 1;

185 186 187 188 189 190 191
    if (first_rec)
    {
        rec_len = sizeof(struct darshan_apmpi_header_record);
        first_rec = 0;
    }
    else
        rec_len = sizeof(struct darshan_apmpi_perf_record);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
192
    
193
    ret = darshan_log_put_mod(fd, DARSHAN_APMPI_MOD, buf,
Sudheer Chunduri's avatar
Sudheer Chunduri committed
194
                              rec_len, APMPI_VER);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
195 196 197 198 199 200 201 202 203 204
    if(ret < 0)
        return(-1);

    return(0);
}

static void darshan_log_print_apmpi_rec(void *rec, char *file_name,
    char *mnt_pt, char *fs_type)
{
    int i;
205
    static int first_rec = 1;
206
    static int sync_flag;
207
    struct darshan_apmpi_header_record *hdr_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
208
    struct darshan_apmpi_perf_record *prf_rec;
209 210 211 212
    
    if (first_rec)
    {
        hdr_rec = rec;
213
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
214 215 216
            hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
            "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpitime,
            "", "", "");
217
        if(hdr_rec->sync_flag)
218
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
219 220 221
            hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
            "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpisynctime,
            "", "", "");
222
        first_rec = 0;
223
        sync_flag = hdr_rec->sync_flag;
224 225 226
    }
    else
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
227
        prf_rec = rec;
228

229
        DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
230
                  prf_rec->base_rec.rank, prf_rec->base_rec.id,
231
                  "nodeid", prf_rec->node_name,
232 233
                  "", "", "");
   
Sudheer Chunduri's avatar
Sudheer Chunduri committed
234 235
        for(i = 0; i < APMPI_NUM_INDICES; i++)
        {
236
            DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
237 238 239 240
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
                apmpi_counter_names[i], prf_rec->counters[i],
                "", "", "");
        }
241
        for(i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
242
        {
243
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
244
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
245 246 247
                apmpi_f_mpiop_totaltime_counter_names[i], prf_rec->fcounters[i],
                "", "", "");
        }
248
        if(sync_flag)
249
        {
250 251 252 253 254 255 256
            for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
            {
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
                    prf_rec->base_rec.rank, prf_rec->base_rec.id,
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec->fsynccounters[i],
                    "", "", "");
            }
257
        }
258 259 260
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
            prf_rec->base_rec.rank, prf_rec->base_rec.id,
            apmpi_f_mpi_global_counter_names[0], prf_rec->fglobalcounters[0],
261
                "", "", "");
262 263
        if(sync_flag)
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
264 265
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
                apmpi_f_mpi_global_counter_names[1], prf_rec->fglobalcounters[1],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
266
                "", "", "");
267
    }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
268 269 270 271 272 273 274

    return;
}

static void darshan_log_print_apmpi_description(int ver)
{
    printf("\n# description of APMPI counters: %d\n", ver);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
275 276 277
    //printf("#     node:    node connected to this router\n");
    //printf("#     AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCz of x y tile\n");
    //printf("#     AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n");
Sudheer Chunduri's avatar
Sudheer Chunduri committed
278 279 280 281 282 283 284

    return;
}

static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
    void *file_rec2, char *file_name2)
{
285 286
    struct darshan_apmpi_header_record *hdr_rec1;
    struct darshan_apmpi_header_record *hdr_rec2;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
287 288 289
    struct darshan_apmpi_perf_record   *prf_rec1;
    struct darshan_apmpi_perf_record   *prf_rec2;

290 291 292
    hdr_rec1 = (struct darshan_apmpi_header_record*) file_rec1;
    hdr_rec2 = (struct darshan_apmpi_header_record*) file_rec2;
    prf_rec1 = (struct darshan_apmpi_perf_record*) file_rec1;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
293
    prf_rec2 = (struct darshan_apmpi_perf_record*) file_rec2;
294 295 296
    static int sync_flag;
    sync_flag = hdr_rec1->sync_flag && hdr_rec2->sync_flag;

297 298 299 300 301 302
    if (hdr_rec1->magic == APMPI_MAGIC)
    {
        /* this is the header record */   
        if (!hdr_rec2) 
        {
            printf("- ");   
303
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
304 305 306
                hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
                "", "", "");
307
            if(sync_flag)
308
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
309 310 311
                hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
                "", "", "");
312 313 314 315
        }
        else if (!hdr_rec1)
        {
            printf("+ ");
316
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
317 318 319
                hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
                "", "", "");
320
            if(sync_flag)
321
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
322 323 324
                hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
                "", "", "");
325 326 327 328 329 330
        }
        else
        {   
            if (hdr_rec1->apmpi_f_variance_total_mpitime != hdr_rec2->apmpi_f_variance_total_mpitime)
            {
                printf("- ");
331
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
332 333 334 335
                    hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                    "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
                    "", "", "");
                printf("+ ");
336
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
337 338 339 340
                    hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                    "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
                    "", "", "");
            }
341 342
            if(sync_flag)
            {
343 344 345
            if (hdr_rec1->apmpi_f_variance_total_mpisynctime != hdr_rec2->apmpi_f_variance_total_mpisynctime)
            {
                printf("- ");
346
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
347 348 349 350
                    hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                    "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
                    "", "", "");
                printf("+ ");
351
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
352 353 354 355
                    hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                    "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
                    "", "", "");
            }
356
            }
357 358 359 360
        }
    }
    else
    {   
361 362 363
        if(!prf_rec2)
        {
            printf("- ");
364
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
365
                  prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
366
                  "nodeid", prf_rec1->node_name,
367 368 369 370 371
                  "", "", "");
        }
        else if (!prf_rec1)
        {
            printf("+ ");
372
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
373
                  prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
374
                  "nodeid", prf_rec2->node_name,
375 376
                  "", "", "");
        }
377
        else if (prf_rec1->node_name != prf_rec2->node_name)
378 379
        {
            printf("- ");
380
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
381
                  prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
382
                  "nodeid", prf_rec1->node_name,
383 384
                  "", "", "");
            printf("+ ");
385
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
386
                  prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
387
                  "nodeid", prf_rec2->node_name,
388 389
                  "", "", "");
        }
390 391 392 393 394 395
        int i;
        for(i = 0; i < APMPI_NUM_INDICES; i++)
        {
            if (!prf_rec2)
            {   
                printf("- ");
396
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
397 398 399 400 401 402 403
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_counter_names[i], prf_rec1->counters[i],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
404
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
405 406 407 408 409 410 411
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_counter_names[i], prf_rec2->counters[i],
                    "", "", "");
            }
            else if (prf_rec1->counters[i] != prf_rec2->counters[i])
            {
                printf("- ");
412
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
413 414 415 416
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_counter_names[i], prf_rec1->counters[i],
                    "", "", "");
                printf("+ ");
417
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
418 419 420 421 422 423
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_counter_names[i], prf_rec2->counters[i],
                    "", "", "");
            }
        }
        
424
        for(i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
425 426 427 428
        {   
            if (!prf_rec2)
            {   
                printf("- ");
429
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
430 431 432 433 434 435 436
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
437
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
438 439 440 441 442 443 444
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
                    "", "", "");
            }
            else if (prf_rec1->counters[i] != prf_rec2->counters[i])
            {
                printf("- ");
445
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
446 447 448 449
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
                    "", "", "");
                printf("+ ");
450
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
451 452 453 454 455
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
                    "", "", "");
            }
        }
456 457
        if(sync_flag)
        {
458
        for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
459 460 461 462
        {
            if (!prf_rec2)
            {
                printf("- ");
463
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
464
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
465
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
466 467 468 469 470 471
                    "", "", "");

            }
            else if (!prf_rec1)
            {
                printf("+ ");
472
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
473
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
474
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
475 476 477 478 479
                    "", "", "");
            }
            else if (prf_rec1->fsynccounters[i] != prf_rec2->fsynccounters[i])
            {
                printf("- ");
480
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
481
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
482
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
483 484
                    "", "", "");
                printf("+ ");
485
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
486
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
487
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
488 489 490
                    "", "", "");
            }
        }
491 492
        }
        //for(i = 0; i < APMPI_F_MPI_GLOBAL_NUM_INDICES; i++)
493 494 495 496
        {
            if (!prf_rec2)
            {
                printf("- ");
497
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
498
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
499
                    apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
500 501 502 503 504
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
505
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
506
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
507
                    apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
508 509
                    "", "", "");
            }
510
            else if (prf_rec1->fglobalcounters[0] != prf_rec2->fglobalcounters[0])
511 512
            {
                printf("- ");
513
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
514
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
515
                    apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
516 517
                    "", "", "");
                printf("+ ");
518
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
519
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
520
                    apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
521 522
                    "", "", "");
            }
523 524 525 526 527
            if(sync_flag)
            {
            if (!prf_rec2)
            {
                printf("- ");
528
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
529 530 531 532 533 534 535
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec1->fglobalcounters[1],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
536
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
537 538 539 540 541 542 543
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec2->fglobalcounters[1],
                    "", "", "");
            }
            else if (prf_rec1->fglobalcounters[1] != prf_rec2->fglobalcounters[1])
            {
                printf("- ");
544
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
545 546 547 548
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec1->fglobalcounters[1],
                    "", "", "");
                printf("+ ");
549
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
550 551 552 553 554 555
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec2->fglobalcounters[1],
                    "", "", "");
            }
           }
       }
556
    }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
557 558 559 560 561 562 563 564 565 566 567 568


    return;
}
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */