darshan-apmpi-logutils.c 20.4 KB
Newer Older
Sudheer Chunduri's avatar
Sudheer Chunduri committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (C) 2018 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#define _GNU_SOURCE
#include "darshan-util-config.h"
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <unistd.h>
#include <inttypes.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>

#include "darshan-logutils.h"
#include "darshan-apmpi-log-format.h"

Sudheer Chunduri's avatar
Sudheer Chunduri committed
23
/* counter name strings for the MPI module */
Sudheer Chunduri's avatar
Sudheer Chunduri committed
24 25 26
#define Y(a) #a, 
#define Z(a) #a
#define X I
27
#define V J
Sudheer Chunduri's avatar
Sudheer Chunduri committed
28
char *apmpi_counter_names[] = {
29
    APMPI_MPIOP_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
30
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
31
#undef X
32
#undef V
33 34
#define X F_TIME
#define V F_TIME
35 36 37 38
char *apmpi_f_mpiop_totaltime_counter_names[] = {
    APMPI_F_MPIOP_TOTALTIME_COUNTERS
};
#undef X
39
#undef V
40
#define X F_SYNC
41
#define V F_SYNC
42
char *apmpi_f_mpiop_synctime_counter_names[] = {
43 44 45
    APMPI_F_MPIOP_SYNCTIME_COUNTERS
};
#undef X
46
#undef V
47 48
char *apmpi_f_mpi_global_counter_names[] = {
    APMPI_F_MPI_GLOBAL_COUNTERS
Sudheer Chunduri's avatar
Sudheer Chunduri committed
49
};
Sudheer Chunduri's avatar
Sudheer Chunduri committed
50 51 52
#undef Y
#undef Z

Sudheer Chunduri's avatar
Sudheer Chunduri committed
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p);
static int darshan_log_put_apmpi_rec(darshan_fd fd, void* buf);
static void darshan_log_print_apmpi_rec(void *file_rec,
    char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_print_apmpi_description(int ver);
static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
    void *file_rec2, char *file_name2);

struct darshan_mod_logutil_funcs apmpi_logutils =
{
    .log_get_record = &darshan_log_get_apmpi_rec,
    .log_put_record = &darshan_log_put_apmpi_rec,
    .log_print_record = &darshan_log_print_apmpi_rec,
    .log_print_description = &darshan_log_print_apmpi_description,
    .log_print_diff = &darshan_log_print_apmpi_rec_diff,
    .log_agg_records = NULL
};

71

Sudheer Chunduri's avatar
Sudheer Chunduri committed
72 73
static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p)
{
74
    struct darshan_apmpi_header_record *hdr_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
75 76 77 78 79
    struct darshan_apmpi_perf_record *prf_rec;
    int rec_len;
    char *buffer;
    int i;
    int ret = -1;
80
    static int first_rec = 1;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
81

82
    if(fd->mod_map[DARSHAN_APMPI_MOD].len == 0)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
83 84
        return(0);

85 86 87 88 89 90 91 92
    if(fd->mod_ver[DARSHAN_APMPI_MOD] == 0 ||
        fd->mod_ver[DARSHAN_APMPI_MOD] > APMPI_VER)
    {
        fprintf(stderr, "Error: Invalid APMPI module version number (got %d)\n",
            fd->mod_ver[DARSHAN_APMPI_MOD]);
        return(-1);
    }

Sudheer Chunduri's avatar
Sudheer Chunduri committed
93 94 95 96 97 98 99 100 101 102 103 104 105 106
    if (!*buf_p)
    {
        /* assume this is the largest possible record size */
        buffer = malloc(sizeof(struct darshan_apmpi_perf_record));
        if (!buffer)
        {
            return(-1);
        }
    }
    else
    {
        buffer = *buf_p;
    }

107
    if (fd->mod_ver[DARSHAN_APMPI_MOD] < APMPI_VER)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
108 109 110 111 112
    {
        /* perform conversion as needed */
    }

    /* v1, current version */
113
    if (fd->mod_ver[DARSHAN_APMPI_MOD] == APMPI_VER)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
114
    {
115 116 117 118 119 120
        if (first_rec)
        {
            rec_len = sizeof(struct darshan_apmpi_header_record);
            first_rec = 0;
        }
        else
Sudheer Chunduri's avatar
Sudheer Chunduri committed
121 122
            rec_len = sizeof(struct darshan_apmpi_perf_record);

123
        ret = darshan_log_get_mod(fd, DARSHAN_APMPI_MOD, buffer, rec_len);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
124 125 126 127 128 129
    }

    if (ret == rec_len)
    {
        if(fd->swap_flag)
        {
130 131 132 133 134 135
            if (rec_len == sizeof(struct darshan_apmpi_header_record))
            {
                hdr_rec = (struct darshan_apmpi_header_record*)buffer;
                /* swap bytes if necessary */
                DARSHAN_BSWAP64(&(hdr_rec->base_rec.id));
                DARSHAN_BSWAP64(&(hdr_rec->base_rec.rank));
Shane Snyder's avatar
Shane Snyder committed
136 137
                DARSHAN_BSWAP64(&(hdr_rec->magic));
                DARSHAN_BSWAP32(&(hdr_rec->sync_flag));
138
                DARSHAN_BSWAP64(&(hdr_rec->apmpi_f_variance_total_mpitime));
139
                DARSHAN_BSWAP64(&(hdr_rec->apmpi_f_variance_total_mpisynctime));
140 141 142
            }
            else
            {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
143 144 145 146 147 148 149
                prf_rec = (struct darshan_apmpi_perf_record*)buffer;
                DARSHAN_BSWAP64(&(prf_rec->base_rec.id));
                DARSHAN_BSWAP64(&(prf_rec->base_rec.rank));
                for (i = 0; i < APMPI_NUM_INDICES; i++)
                {
                    DARSHAN_BSWAP64(&prf_rec->counters[i]);
                }
150
                for (i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
151 152 153
                {
                    DARSHAN_BSWAP64(&prf_rec->fcounters[i]);
                }
154
                for (i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
155 156 157
                {
                    DARSHAN_BSWAP64(&prf_rec->fsynccounters[i]);
                }
158
                for (i = 0; i < APMPI_F_MPI_GLOBAL_NUM_INDICES; i++)
159 160 161 162
                {
                    DARSHAN_BSWAP64(&prf_rec->fglobalcounters[i]);
                }
            }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
163 164 165 166 167 168
        }
        *buf_p = buffer;
        return(1);
    }
    else if (ret < 0)
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
169
 //       *buf_p = NULL;
170
        if (!*buf_p) free(buffer);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
171 172 173 174
        return(-1);
    }
    else
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
175
  //      *buf_p = NULL;
176
        if (!*buf_p) free(buffer);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
177 178 179 180 181 182 183 184 185 186
        return(0);
    }
}

static int darshan_log_put_apmpi_rec(darshan_fd fd, void* buf)
{
    int ret;
    int rec_len;
    static int first_rec = 1;

187 188 189 190 191 192 193
    if (first_rec)
    {
        rec_len = sizeof(struct darshan_apmpi_header_record);
        first_rec = 0;
    }
    else
        rec_len = sizeof(struct darshan_apmpi_perf_record);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
194
    
195
    ret = darshan_log_put_mod(fd, DARSHAN_APMPI_MOD, buf,
Sudheer Chunduri's avatar
Sudheer Chunduri committed
196
                              rec_len, APMPI_VER);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
197 198 199 200 201 202 203 204 205 206
    if(ret < 0)
        return(-1);

    return(0);
}

static void darshan_log_print_apmpi_rec(void *rec, char *file_name,
    char *mnt_pt, char *fs_type)
{
    int i;
207
    static int first_rec = 1;
208
    static int sync_flag;
209
    struct darshan_apmpi_header_record *hdr_rec;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
210
    struct darshan_apmpi_perf_record *prf_rec;
211 212 213 214
    
    if (first_rec)
    {
        hdr_rec = rec;
215
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
216 217 218
            hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
            "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpitime,
            "", "", "");
219
        if(hdr_rec->sync_flag)
220
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
221 222 223
            hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
            "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpisynctime,
            "", "", "");
224
        first_rec = 0;
225
        sync_flag = hdr_rec->sync_flag;
226 227 228
    }
    else
    {
Sudheer Chunduri's avatar
Sudheer Chunduri committed
229
        prf_rec = rec;
230

231
        DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
232
                  prf_rec->base_rec.rank, prf_rec->base_rec.id,
233
                  "nodeid", prf_rec->node_name,
234 235
                  "", "", "");
   
Sudheer Chunduri's avatar
Sudheer Chunduri committed
236 237
        for(i = 0; i < APMPI_NUM_INDICES; i++)
        {
238
            DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
239 240 241 242
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
                apmpi_counter_names[i], prf_rec->counters[i],
                "", "", "");
        }
243
        for(i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
Sudheer Chunduri's avatar
Sudheer Chunduri committed
244
        {
245
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
246
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
247 248 249
                apmpi_f_mpiop_totaltime_counter_names[i], prf_rec->fcounters[i],
                "", "", "");
        }
250
        if(sync_flag)
251
        {
252 253 254 255 256 257 258
            for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
            {
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
                    prf_rec->base_rec.rank, prf_rec->base_rec.id,
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec->fsynccounters[i],
                    "", "", "");
            }
259
        }
260 261 262
        DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
            prf_rec->base_rec.rank, prf_rec->base_rec.id,
            apmpi_f_mpi_global_counter_names[0], prf_rec->fglobalcounters[0],
263
                "", "", "");
264 265
        if(sync_flag)
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
266 267
                prf_rec->base_rec.rank, prf_rec->base_rec.id,
                apmpi_f_mpi_global_counter_names[1], prf_rec->fglobalcounters[1],
Sudheer Chunduri's avatar
Sudheer Chunduri committed
268
                "", "", "");
269
    }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
270 271 272 273 274 275 276

    return;
}

static void darshan_log_print_apmpi_description(int ver)
{
    printf("\n# description of APMPI counters: %d\n", ver);
Sudheer Chunduri's avatar
Sudheer Chunduri committed
277 278 279
    //printf("#     node:    node connected to this router\n");
    //printf("#     AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCz of x y tile\n");
    //printf("#     AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n");
Sudheer Chunduri's avatar
Sudheer Chunduri committed
280 281 282 283 284 285 286

    return;
}

static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
    void *file_rec2, char *file_name2)
{
287 288
    struct darshan_apmpi_header_record *hdr_rec1;
    struct darshan_apmpi_header_record *hdr_rec2;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
289 290 291
    struct darshan_apmpi_perf_record   *prf_rec1;
    struct darshan_apmpi_perf_record   *prf_rec2;

292 293 294
    hdr_rec1 = (struct darshan_apmpi_header_record*) file_rec1;
    hdr_rec2 = (struct darshan_apmpi_header_record*) file_rec2;
    prf_rec1 = (struct darshan_apmpi_perf_record*) file_rec1;
Sudheer Chunduri's avatar
Sudheer Chunduri committed
295
    prf_rec2 = (struct darshan_apmpi_perf_record*) file_rec2;
296 297 298
    static int sync_flag;
    sync_flag = hdr_rec1->sync_flag && hdr_rec2->sync_flag;

299 300 301 302 303 304
    if (hdr_rec1->magic == APMPI_MAGIC)
    {
        /* this is the header record */   
        if (!hdr_rec2) 
        {
            printf("- ");   
305
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
306 307 308
                hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
                "", "", "");
309
            if(sync_flag)
310
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
311 312 313
                hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
                "", "", "");
314 315 316 317
        }
        else if (!hdr_rec1)
        {
            printf("+ ");
318
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
319 320 321
                hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
                "", "", "");
322
            if(sync_flag)
323
            DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
324 325 326
                hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
                "", "", "");
327 328 329 330 331 332
        }
        else
        {   
            if (hdr_rec1->apmpi_f_variance_total_mpitime != hdr_rec2->apmpi_f_variance_total_mpitime)
            {
                printf("- ");
333
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
334 335 336 337
                    hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                    "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
                    "", "", "");
                printf("+ ");
338
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
339 340 341 342
                    hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                    "RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
                    "", "", "");
            }
343 344
            if(sync_flag)
            {
345 346 347
            if (hdr_rec1->apmpi_f_variance_total_mpisynctime != hdr_rec2->apmpi_f_variance_total_mpisynctime)
            {
                printf("- ");
348
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
349 350 351 352
                    hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
                    "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
                    "", "", "");
                printf("+ ");
353
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
354 355 356 357
                    hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
                    "RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
                    "", "", "");
            }
358
            }
359 360 361 362
        }
    }
    else
    {   
363 364 365
        if(!prf_rec2)
        {
            printf("- ");
366
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
367
                  prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
368
                  "nodeid", prf_rec1->node_name,
369 370 371 372 373
                  "", "", "");
        }
        else if (!prf_rec1)
        {
            printf("+ ");
374
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
375
                  prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
376
                  "nodeid", prf_rec2->node_name,
377 378
                  "", "", "");
        }
379
        else if (prf_rec1->node_name != prf_rec2->node_name)
380 381
        {
            printf("- ");
382
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
383
                  prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
384
                  "nodeid", prf_rec1->node_name,
385 386
                  "", "", "");
            printf("+ ");
387
            DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
388
                  prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
389
                  "nodeid", prf_rec2->node_name,
390 391
                  "", "", "");
        }
392 393 394 395 396 397
        int i;
        for(i = 0; i < APMPI_NUM_INDICES; i++)
        {
            if (!prf_rec2)
            {   
                printf("- ");
398
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
399 400 401 402 403 404 405
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_counter_names[i], prf_rec1->counters[i],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
406
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
407 408 409 410 411 412 413
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_counter_names[i], prf_rec2->counters[i],
                    "", "", "");
            }
            else if (prf_rec1->counters[i] != prf_rec2->counters[i])
            {
                printf("- ");
414
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
415 416 417 418
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_counter_names[i], prf_rec1->counters[i],
                    "", "", "");
                printf("+ ");
419
                DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
420 421 422 423 424 425
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_counter_names[i], prf_rec2->counters[i],
                    "", "", "");
            }
        }
        
426
        for(i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
427 428 429 430
        {   
            if (!prf_rec2)
            {   
                printf("- ");
431
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
432 433 434 435 436 437 438
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
439
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
440 441 442 443 444 445 446
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
                    "", "", "");
            }
            else if (prf_rec1->counters[i] != prf_rec2->counters[i])
            {
                printf("- ");
447
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
448 449 450 451
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
                    "", "", "");
                printf("+ ");
452
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
453 454 455 456 457
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
                    "", "", "");
            }
        }
458 459
        if(sync_flag)
        {
460
        for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
461 462 463 464
        {
            if (!prf_rec2)
            {
                printf("- ");
465
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
466
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
467
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
468 469 470 471 472 473
                    "", "", "");

            }
            else if (!prf_rec1)
            {
                printf("+ ");
474
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
475
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
476
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
477 478 479 480 481
                    "", "", "");
            }
            else if (prf_rec1->fsynccounters[i] != prf_rec2->fsynccounters[i])
            {
                printf("- ");
482
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
483
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
484
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
485 486
                    "", "", "");
                printf("+ ");
487
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
488
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
489
                    apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
490 491 492
                    "", "", "");
            }
        }
493 494
        }
        //for(i = 0; i < APMPI_F_MPI_GLOBAL_NUM_INDICES; i++)
495 496 497 498
        {
            if (!prf_rec2)
            {
                printf("- ");
499
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
500
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
501
                    apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
502 503 504 505 506
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
507
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
508
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
509
                    apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
510 511
                    "", "", "");
            }
512
            else if (prf_rec1->fglobalcounters[0] != prf_rec2->fglobalcounters[0])
513 514
            {
                printf("- ");
515
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
516
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
517
                    apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
518 519
                    "", "", "");
                printf("+ ");
520
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
521
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
522
                    apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
523 524
                    "", "", "");
            }
525 526 527 528 529
            if(sync_flag)
            {
            if (!prf_rec2)
            {
                printf("- ");
530
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
531 532 533 534 535 536 537
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec1->fglobalcounters[1],
                    "", "", "");
            }
            else if (!prf_rec1)
            {
                printf("+ ");
538
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
539 540 541 542 543 544 545
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec2->fglobalcounters[1],
                    "", "", "");
            }
            else if (prf_rec1->fglobalcounters[1] != prf_rec2->fglobalcounters[1])
            {
                printf("- ");
546
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
547 548 549 550
                    prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec1->fglobalcounters[1],
                    "", "", "");
                printf("+ ");
551
                DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
552 553 554 555 556 557
                    prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
                    apmpi_f_mpi_global_counter_names[1], prf_rec2->fglobalcounters[1],
                    "", "", "");
            }
           }
       }
558
    }
Sudheer Chunduri's avatar
Sudheer Chunduri committed
559 560 561 562 563 564 565 566 567 568 569 570


    return;
}
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ts=8 sts=4 sw=4 expandtab
 */