Commit ba69643f authored by Sudheer Chunduri's avatar Sudheer Chunduri
Browse files

Merge branch 'ssnyder-updates' into 'master'

autoperf updates for inclusion in Darshan 3.3.0 release

See merge request AutoPerf/autoperf!3
parents 3b14dfcd f70f5bb7
......@@ -13,11 +13,8 @@ CFLAGS_SHARED += \
-DDARSHAN_USE_APMPI \
-I$(srcdir)/../modules/autoperf/apmpi
lib/darshan-apmpi.o: lib/darshan-apmpi.c lib/darshan-apmpi-utils.h darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apmpi-log-format.h | lib
lib/darshan-apmpi.o: lib/darshan-apmpi.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apmpi-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-apmpi.po: lib/darshan-apmpi.c lib/darshan-apmpi-utils.h darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apmpi-log-format.h | lib
lib/darshan-apmpi.po: lib/darshan-apmpi.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apmpi-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
print-apmpi-size: print-apmpi-size.c $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apmpi/darshan-apmpi-log-format.h
$(CC) $(CFLAGS) -g -O0 -o $@ $<
#ifndef __APMPI_UTILS_H__
#define __APMPI_UTILS_H__
#include <regex.h>
static void search_hwinfo(const char * mstr, char *mode)
{
FILE *f;
int r;
char *fdata;
long len;
regex_t preg;
regmatch_t mreg[1];
r = regcomp(&preg, mstr, 0);
f = fopen ("/.hwinfo.cray", "rb");
fseek(f, 0, SEEK_END);
len = ftell(f);
fdata = malloc(len);
fseek(f, 0, SEEK_SET);
fread(fdata, sizeof(char), len, f);
fclose(f);
r = regexec(&preg, fdata, 1, mreg, 0);
if ((r == 0) && (mreg[0].rm_so > -1))
{
sscanf(fdata+mreg[0].rm_so+strlen(mstr)-2, "%s", mode);
}
regfree(&preg);
free(fdata);
return;
}
static unsigned int count_bits(unsigned int *bitvec, int cnt)
{
unsigned int count = 0;
int i;
for (i = 0; i < cnt; i++)
{
count += __builtin_popcount(bitvec[i]);
}
return count;
}
#endif
......@@ -13,15 +13,12 @@
#include <stdlib.h>
#include <pthread.h>
#include <assert.h>
#include <papi.h>
#include "uthash.h"
#include "darshan.h"
#include "darshan-dynamic.h"
#include "darshan-apmpi-log-format.h"
#include "darshan-apmpi-utils.h"
typedef long long ap_bytes_t;
#define MAX(x,y) ((x>y)?x:y)
#define MIN(x,y) ((x==0.0)?y:((x<y)?x:y))
......@@ -274,7 +271,7 @@ static pthread_mutex_t apmpi_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER
static int my_rank = -1;
/* internal helper functions for the apmpi module */
void apmpi_runtime_initialize(void);
static void apmpi_runtime_initialize(void);
/* forward declaration for shutdown function needed to interface with darshan-core */
#ifdef HAVE_MPI
......@@ -298,7 +295,6 @@ static void apmpi_shutdown(
static void initialize_counters (void)
{
int i;
int code = 0;
for (i = 0; i < APMPI_NUM_INDICES; i++)
{
apmpi_runtime->perf_record->counters[i] = 0;
......@@ -338,7 +334,7 @@ static void capture(struct darshan_apmpi_perf_record *rec,
return;
}
void apmpi_runtime_initialize()
static void apmpi_runtime_initialize()
{
int apmpi_buf_size;
char rec_name[128];
......@@ -364,25 +360,17 @@ void apmpi_runtime_initialize()
/* register the apmpi module with the darshan-core component */
darshan_core_register_module(
APMPI_MOD,
DARSHAN_APMPI_MOD,
mod_funcs,
&apmpi_buf_size,
&my_rank,
NULL);
/* not enough memory to fit apmpi module record */
if(apmpi_buf_size < sizeof(struct darshan_apmpi_header_record) + sizeof(struct darshan_apmpi_perf_record))
{
darshan_core_unregister_module(APMPI_MOD);
APMPI_UNLOCK();
return;
}
/* initialize module's global state */
apmpi_runtime = malloc(sizeof(*apmpi_runtime));
if(!apmpi_runtime)
{
darshan_core_unregister_module(APMPI_MOD);
darshan_core_unregister_module(DARSHAN_APMPI_MOD);
APMPI_UNLOCK();
return;
}
......@@ -397,12 +385,12 @@ void apmpi_runtime_initialize()
apmpi_runtime->header_id,
//NULL,
"darshan-apmpi-header",
APMPI_MOD,
DARSHAN_APMPI_MOD,
sizeof(struct darshan_apmpi_header_record),
NULL);
if(!(apmpi_runtime->header_record))
{
darshan_core_unregister_module(APMPI_MOD);
darshan_core_unregister_module(DARSHAN_APMPI_MOD);
free(apmpi_runtime);
apmpi_runtime = NULL;
APMPI_UNLOCK();
......@@ -424,12 +412,12 @@ void apmpi_runtime_initialize()
apmpi_runtime->perf_record = darshan_core_register_record(
apmpi_runtime->rec_id,
"APMPI",
APMPI_MOD,
DARSHAN_APMPI_MOD,
sizeof(struct darshan_apmpi_perf_record),
NULL);
if(!(apmpi_runtime->perf_record))
{
darshan_core_unregister_module(APMPI_MOD);
darshan_core_unregister_module(DARSHAN_APMPI_MOD);
free(apmpi_runtime);
apmpi_runtime = NULL;
APMPI_UNLOCK();
......@@ -643,40 +631,40 @@ static void apmpi_shutdown(
#define APMPI_RECORD_UPDATE(MPI_OP) do { \
if(ret != MPI_SUCCESS) break; \
apmpi_runtime->perf_record->counters[Y(MPI_OP ## _CALL_COUNT)]++; \
apmpi_runtime->perf_record->counters[Y(MPI_OP ## _TOTAL_BYTES)] += bytes; \
DARSHAN_MSG_BUCKET_INC(&(apmpi_runtime->perf_record->counters[Y(MPI_OP ## _MSG_SIZE_AGG_0_256)]), bytes); \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _TOTAL_TIME)] += tdiff; \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
apmpi_runtime->perf_record->counters[MPI_OP ## _CALL_COUNT]++; \
apmpi_runtime->perf_record->counters[MPI_OP ## _TOTAL_BYTES] += bytes; \
DARSHAN_MSG_BUCKET_INC(&(apmpi_runtime->perf_record->counters[MPI_OP ## _MSG_SIZE_AGG_0_256]), bytes); \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _TOTAL_TIME] += tdiff; \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MAX_TIME] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MIN_TIME] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
} while(0)
#define APMPI_RECORD_UPDATE_NOMSG(MPI_OP) do { \
if(ret != MPI_SUCCESS) break; \
apmpi_runtime->perf_record->counters[Y(MPI_OP ## _CALL_COUNT)]++; \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _TOTAL_TIME)] += tdiff; \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
apmpi_runtime->perf_record->counters[MPI_OP ## _CALL_COUNT]++; \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _TOTAL_TIME] += tdiff; \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MAX_TIME] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MIN_TIME] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
} while(0)
#define APMPI_RECORD_UPDATE_SYNC(MPI_OP) do { \
if(ret != MPI_SUCCESS) break; \
apmpi_runtime->perf_record->counters[Y(MPI_OP ## _CALL_COUNT)]++; \
apmpi_runtime->perf_record->counters[Y(MPI_OP ## _TOTAL_BYTES)] += bytes; \
DARSHAN_MSG_BUCKET_INC(&(apmpi_runtime->perf_record->counters[Y(MPI_OP ## _MSG_SIZE_AGG_0_256)]), bytes); \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _TOTAL_TIME)] += tdiff; \
apmpi_runtime->perf_record->fsynccounters[Y(MPI_OP ## _TOTAL_SYNC_TIME)] += tsync; \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
apmpi_runtime->perf_record->counters[MPI_OP ## _CALL_COUNT]++; \
apmpi_runtime->perf_record->counters[MPI_OP ## _TOTAL_BYTES] += bytes; \
DARSHAN_MSG_BUCKET_INC(&(apmpi_runtime->perf_record->counters[MPI_OP ## _MSG_SIZE_AGG_0_256]), bytes); \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _TOTAL_TIME] += tdiff; \
apmpi_runtime->perf_record->fsynccounters[MPI_OP ## _TOTAL_SYNC_TIME] += tsync; \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MAX_TIME] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MIN_TIME] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
} while(0)
#define APMPI_RECORD_UPDATE_SYNC_NOMSG(MPI_OP) do { \
if(ret != MPI_SUCCESS) break; \
apmpi_runtime->perf_record->counters[Y(MPI_OP ## _CALL_COUNT)]++; \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _TOTAL_TIME)] += tdiff; \
apmpi_runtime->perf_record->fsynccounters[Y(MPI_OP ## _TOTAL_SYNC_TIME)] += tsync; \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
apmpi_runtime->perf_record->counters[MPI_OP ## _CALL_COUNT]++; \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _TOTAL_TIME] += tdiff; \
apmpi_runtime->perf_record->fsynccounters[MPI_OP ## _TOTAL_SYNC_TIME] += tsync; \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MAX_TIME] = MAX(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MAX_TIME)], tdiff); \
apmpi_runtime->perf_record->fcounters[MPI_OP ## _MIN_TIME] = MIN(apmpi_runtime->perf_record->fcounters[Y(MPI_OP ## _MIN_TIME)], tdiff); \
} while(0)
#define Y(a) a
......
......@@ -21,8 +21,6 @@
#include "darshan-apmpi-log-format.h"
/* counter name strings for the MPI module */
//#define X(a) #a,
#define Y(a) #a,
#define Z(a) #a
#define X I
......@@ -81,9 +79,17 @@ static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p)
int ret = -1;
static int first_rec = 1;
if(fd->mod_map[APMPI_MOD].len == 0)
if(fd->mod_map[DARSHAN_APMPI_MOD].len == 0)
return(0);
if(fd->mod_ver[DARSHAN_APMPI_MOD] == 0 ||
fd->mod_ver[DARSHAN_APMPI_MOD] > APMPI_VER)
{
fprintf(stderr, "Error: Invalid APMPI module version number (got %d)\n",
fd->mod_ver[DARSHAN_APMPI_MOD]);
return(-1);
}
if (!*buf_p)
{
/* assume this is the largest possible record size */
......@@ -98,21 +104,13 @@ static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p)
buffer = *buf_p;
}
if (fd->mod_ver[APMPI_MOD] == 0)
{
printf("Either unknown or debug version: %d\n",
fd->mod_ver[APMPI_MOD]);
return(0);
}
if ((fd->mod_ver[APMPI_MOD] > 0) &&
(fd->mod_ver[APMPI_MOD] < APMPI_VER))
if (fd->mod_ver[DARSHAN_APMPI_MOD] < APMPI_VER)
{
/* perform conversion as needed */
}
/* v1, current version */
if (fd->mod_ver[APMPI_MOD] == APMPI_VER)
if (fd->mod_ver[DARSHAN_APMPI_MOD] == APMPI_VER)
{
if (first_rec)
{
......@@ -122,7 +120,7 @@ static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p)
else
rec_len = sizeof(struct darshan_apmpi_perf_record);
ret = darshan_log_get_mod(fd, APMPI_MOD, buffer, rec_len);
ret = darshan_log_get_mod(fd, DARSHAN_APMPI_MOD, buffer, rec_len);
}
if (ret == rec_len)
......@@ -167,13 +165,13 @@ static int darshan_log_get_apmpi_rec(darshan_fd fd, void** buf_p)
else if (ret < 0)
{
// *buf_p = NULL;
// if (buffer) free(buffer);
if (!*buf_p) free(buffer);
return(-1);
}
else
{
// *buf_p = NULL;
// if (buffer) free(buffer);
if (!*buf_p) free(buffer);
return(0);
}
}
......@@ -192,7 +190,7 @@ static int darshan_log_put_apmpi_rec(darshan_fd fd, void* buf)
else
rec_len = sizeof(struct darshan_apmpi_perf_record);
ret = darshan_log_put_mod(fd, APMPI_MOD, buf,
ret = darshan_log_put_mod(fd, DARSHAN_APMPI_MOD, buf,
rec_len, APMPI_VER);
if(ret < 0)
return(-1);
......@@ -212,12 +210,12 @@ static void darshan_log_print_apmpi_rec(void *rec, char *file_name,
if (first_rec)
{
hdr_rec = rec;
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
"RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpitime,
"", "", "");
if(hdr_rec->sync_flag)
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec->base_rec.rank, hdr_rec->base_rec.id,
"RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec->apmpi_f_variance_total_mpisynctime,
"", "", "");
......@@ -228,40 +226,41 @@ static void darshan_log_print_apmpi_rec(void *rec, char *file_name,
{
prf_rec = rec;
DARSHAN_S_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
"nodeid", prf_rec->node_name,
"", "", "");
for(i = 0; i < APMPI_NUM_INDICES; i++)
{
DARSHAN_U_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
apmpi_counter_names[i], prf_rec->counters[i],
"", "", "");
}
for(i = 0; i < APMPI_F_MPIOP_TOTALTIME_NUM_INDICES; i++)
{
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
apmpi_f_mpiop_totaltime_counter_names[i], prf_rec->fcounters[i],
"", "", "");
}
if(sync_flag){
for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
if(sync_flag)
{
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
apmpi_f_mpiop_synctime_counter_names[i], prf_rec->fsynccounters[i],
"", "", "");
}
for(i = 0; i < APMPI_F_MPIOP_SYNCTIME_NUM_INDICES; i++)
{
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
apmpi_f_mpiop_synctime_counter_names[i], prf_rec->fsynccounters[i],
"", "", "");
}
}
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
apmpi_f_mpi_global_counter_names[0], prf_rec->fglobalcounters[0],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
apmpi_f_mpi_global_counter_names[0], prf_rec->fglobalcounters[0],
"", "", "");
if(sync_flag)
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
if(sync_flag)
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec->base_rec.rank, prf_rec->base_rec.id,
apmpi_f_mpi_global_counter_names[1], prf_rec->fglobalcounters[1],
"", "", "");
......@@ -301,12 +300,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (!hdr_rec2)
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
"RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
"", "", "");
if(sync_flag)
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
"RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
"", "", "");
......@@ -314,12 +313,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (!hdr_rec1)
{
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
"RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
"", "", "");
if(sync_flag)
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
"RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
"", "", "");
......@@ -329,12 +328,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (hdr_rec1->apmpi_f_variance_total_mpitime != hdr_rec2->apmpi_f_variance_total_mpitime)
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
"RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpitime,
"", "", "");
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
"RANKS_TOTAL_MPITIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpitime,
"", "", "");
......@@ -344,12 +343,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (hdr_rec1->apmpi_f_variance_total_mpisynctime != hdr_rec2->apmpi_f_variance_total_mpisynctime)
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id,
"RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec1->apmpi_f_variance_total_mpisynctime,
"", "", "");
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id,
"RANKS_TOTAL_MPISYNCTIME_VARIANCE", hdr_rec2->apmpi_f_variance_total_mpisynctime,
"", "", "");
......@@ -362,7 +361,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if(!prf_rec2)
{
printf("- ");
DARSHAN_S_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
"nodeid", prf_rec1->node_name,
"", "", "");
......@@ -370,7 +369,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (!prf_rec1)
{
printf("+ ");
DARSHAN_S_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
"nodeid", prf_rec2->node_name,
"", "", "");
......@@ -378,12 +377,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (prf_rec1->node_name != prf_rec2->node_name)
{
printf("- ");
DARSHAN_S_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
"nodeid", prf_rec1->node_name,
"", "", "");
printf("+ ");
DARSHAN_S_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_S_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
"nodeid", prf_rec2->node_name,
"", "", "");
......@@ -394,7 +393,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (!prf_rec2)
{
printf("- ");
DARSHAN_U_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_counter_names[i], prf_rec1->counters[i],
"", "", "");
......@@ -402,7 +401,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (!prf_rec1)
{
printf("+ ");
DARSHAN_U_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_counter_names[i], prf_rec2->counters[i],
"", "", "");
......@@ -410,12 +409,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (prf_rec1->counters[i] != prf_rec2->counters[i])
{
printf("- ");
DARSHAN_U_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_counter_names[i], prf_rec1->counters[i],
"", "", "");
printf("+ ");
DARSHAN_U_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_counter_names[i], prf_rec2->counters[i],
"", "", "");
......@@ -427,7 +426,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (!prf_rec2)
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
"", "", "");
......@@ -435,7 +434,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (!prf_rec1)
{
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
"", "", "");
......@@ -443,12 +442,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (prf_rec1->counters[i] != prf_rec2->counters[i])
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_f_mpiop_totaltime_counter_names[i], prf_rec1->fcounters[i],
"", "", "");
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_f_mpiop_totaltime_counter_names[i], prf_rec2->fcounters[i],
"", "", "");
......@@ -461,7 +460,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (!prf_rec2)
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
"", "", "");
......@@ -470,7 +469,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (!prf_rec1)
{
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
"", "", "");
......@@ -478,12 +477,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (prf_rec1->fsynccounters[i] != prf_rec2->fsynccounters[i])
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_f_mpiop_synctime_counter_names[i], prf_rec1->fsynccounters[i],
"", "", "");
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_f_mpiop_synctime_counter_names[i], prf_rec2->fsynccounters[i],
"", "", "");
......@@ -495,7 +494,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (!prf_rec2)
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
"", "", "");
......@@ -503,7 +502,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (!prf_rec1)
{
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
"", "", "");
......@@ -511,12 +510,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (prf_rec1->fglobalcounters[0] != prf_rec2->fglobalcounters[0])
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_f_mpi_global_counter_names[0], prf_rec1->fglobalcounters[0],
"", "", "");
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_f_mpi_global_counter_names[0], prf_rec2->fglobalcounters[0],
"", "", "");
......@@ -526,7 +525,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
if (!prf_rec2)
{
printf("- ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec1->base_rec.rank, prf_rec1->base_rec.id,
apmpi_f_mpi_global_counter_names[1], prf_rec1->fglobalcounters[1],
"", "", "");
......@@ -534,7 +533,7 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (!prf_rec1)
{
printf("+ ");
DARSHAN_F_COUNTER_PRINT(darshan_module_names[APMPI_MOD],
DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_APMPI_MOD],
prf_rec2->base_rec.rank, prf_rec2->base_rec.id,
apmpi_f_mpi_global_counter_names[1], prf_rec2->fglobalcounters[1],
"", "", "");
......@@ -542,12 +541,12 @@ static void darshan_log_print_apmpi_rec_diff(void *file_rec1, char *file_name1,
else if (prf_rec1->fglobalcounters[1] != prf_rec2->fglobalcounters[1])