Skip to content

GitLab

Commit fe340438 authored by Shane Snyder's avatar Shane Snyder
Browse files

add an aggregation routine to module logutils

parent be3912a8
Hide whitespace changes
Inline Side-by-side
Showing with 224 additions and 1 deletion
darshan-util/Makefile.in
View file @ fe340438
......@@ -156,7 +156,7 @@ endif
clean::
rm -f *.o *.po *.a darshan-analyzer darshan-convert darshan-parser jenkins-hash-gen
rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-parser darshan-stitch-logs jenkins-hash-gen
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
......
darshan-util/darshan-bgq-logutils.c
View file @ fe340438
......@@ -40,6 +40,7 @@ struct darshan_mod_logutil_funcs bgq_logutils =
.log_get_record = &darshan_log_get_bgq_rec,
.log_put_record = &darshan_log_put_bgq_rec,
.log_print_record = &darshan_log_print_bgq_rec,
.log_agg_records = NULL /* TODO: how would aggregation work for the BG/Q module ? */
};
static int darshan_log_get_bgq_rec(darshan_fd fd, void* bgq_buf)
......
darshan-util/darshan-hdf5-logutils.c
View file @ fe340438
......@@ -34,12 +34,14 @@ static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf);
static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf);
static void darshan_log_print_hdf5_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_agg_hdf5_files(void *rec, void *agg_rec, int init_flag);
struct darshan_mod_logutil_funcs hdf5_logutils =
{
.log_get_record = &darshan_log_get_hdf5_file,
.log_put_record = &darshan_log_put_hdf5_file,
.log_print_record = &darshan_log_print_hdf5_file,
.log_agg_records = &darshan_log_agg_hdf5_files
};
static int darshan_log_get_hdf5_file(darshan_fd fd, void* hdf5_buf)
......@@ -111,6 +113,12 @@ static void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
return;
}
static void darshan_log_agg_hdf5_files(void *rec, void *agg_rec, int init_flag)
{
return;
}
/*
* Local variables:
* c-indent-level: 4
......
darshan-util/darshan-logutils.h
View file @ fe340438
......@@ -75,6 +75,12 @@ struct darshan_mod_logutil_funcs
char *mnt_pt,
char *fs_type
);
/* combine two records into a single aggregate record */
void (*log_agg_records)(
void *rec,
void *agg_rec,
int init_flag
);
};
extern struct darshan_mod_logutil_funcs *mod_logutils[];
......
darshan-util/darshan-mpiio-logutils.c
View file @ fe340438
......@@ -34,12 +34,14 @@ static int darshan_log_get_mpiio_file(darshan_fd fd, void* mpiio_buf);
static int darshan_log_put_mpiio_file(darshan_fd fd, void* mpiio_buf);
static void darshan_log_print_mpiio_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag);
struct darshan_mod_logutil_funcs mpiio_logutils =
{
.log_get_record = &darshan_log_get_mpiio_file,
.log_put_record = &darshan_log_put_mpiio_file,
.log_print_record = &darshan_log_print_mpiio_file,
.log_agg_records = &darshan_log_agg_mpiio_files
};
static int darshan_log_get_mpiio_file(darshan_fd fd, void* mpiio_buf)
......@@ -111,6 +113,12 @@ static void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
return;
}
static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
{
return;
}
/*
* Local variables:
* c-indent-level: 4
......
darshan-util/darshan-null-logutils.c
View file @ fe340438
......@@ -36,6 +36,7 @@ static int darshan_log_get_null_record(darshan_fd fd, void* null_buf);
static int darshan_log_put_null_record(darshan_fd fd, void* null_buf);
static void darshan_log_print_null_record(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_agg_null_records(void *rec, void *agg_rec, int init_flag);
/* structure storing each function needed for implementing the darshan
* logutil interface. these functions are used for reading, writing, and
......@@ -46,6 +47,7 @@ struct darshan_mod_logutil_funcs null_logutils =
.log_get_record = &darshan_log_get_null_record,
.log_put_record = &darshan_log_put_null_record,
.log_print_record = &darshan_log_print_null_record,
.log_agg_records = &darshan_log_agg_null_records
};
/* retrieve a NULL record from log file descriptor 'fd', storing the
......@@ -131,6 +133,12 @@ static void darshan_log_print_null_record(void *file_rec, char *file_name,
return;
}
static void darshan_log_agg_null_records(void *rec, void *agg_rec, int init_flag)
{
return;
}
/*
* Local variables:
* c-indent-level: 4
......
darshan-util/darshan-pnetcdf-logutils.c
View file @ fe340438
......@@ -34,12 +34,14 @@ static int darshan_log_get_pnetcdf_file(darshan_fd fd, void* pnetcdf_buf);
static int darshan_log_put_pnetcdf_file(darshan_fd fd, void* pnetcdf_buf);
static void darshan_log_print_pnetcdf_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_agg_pnetcdf_files(void *rec, void *agg_rec, int init_flag);
struct darshan_mod_logutil_funcs pnetcdf_logutils =
{
.log_get_record = &darshan_log_get_pnetcdf_file,
.log_put_record = &darshan_log_put_pnetcdf_file,
.log_print_record = &darshan_log_print_pnetcdf_file,
.log_agg_records = &darshan_log_agg_pnetcdf_files
};
static int darshan_log_get_pnetcdf_file(darshan_fd fd, void* pnetcdf_buf)
......@@ -111,6 +113,12 @@ static void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
return;
}
static void darshan_log_agg_pnetcdf_files(void *rec, void *agg_rec, int init_flag)
{
return;
}
/*
* Local variables:
* c-indent-level: 4
......
darshan-util/darshan-posix-logutils.c
View file @ fe340438
......@@ -34,12 +34,14 @@ static int darshan_log_get_posix_file(darshan_fd fd, void* posix_buf);
static int darshan_log_put_posix_file(darshan_fd fd, void* posix_buf);
static void darshan_log_print_posix_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type);
static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag);
struct darshan_mod_logutil_funcs posix_logutils =
{
.log_get_record = &darshan_log_get_posix_file,
.log_put_record = &darshan_log_put_posix_file,
.log_print_record = &darshan_log_print_posix_file,
.log_agg_records = &darshan_log_agg_posix_files
};
static int darshan_log_get_posix_file(darshan_fd fd, void* posix_buf)
......@@ -111,6 +113,188 @@ static void darshan_log_print_posix_file(void *file_rec, char *file_name,
return;
}
static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
{
struct darshan_posix_file *psx_rec = (struct darshan_posix_file *)rec;
struct darshan_posix_file *agg_psx_rec = (struct darshan_posix_file *)agg_rec;
int i;
double psx_time = psx_rec->fcounters[POSIX_F_READ_TIME] +
psx_rec->fcounters[POSIX_F_WRITE_TIME] +
psx_rec->fcounters[POSIX_F_META_TIME];
/* special case initialization of shared record for
* first call of this function
*/
if(init_flag)
{
/* set fastest/slowest rank counters according to root rank.
* these counters will be determined as the aggregation progresses.
*/
agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] =
psx_rec->counters[POSIX_BYTES_READ] +
psx_rec->counters[POSIX_BYTES_WRITTEN];
agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
agg_psx_rec->counters[POSIX_SLOWEST_RANK] =
agg_psx_rec->counters[POSIX_FASTEST_RANK];
agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] =
agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES];
agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] =
agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME];
}
for(i = 0; i < POSIX_NUM_INDICES; i++)
{
switch(i)
{
case POSIX_OPENS:
case POSIX_READS:
case POSIX_WRITES:
case POSIX_SEEKS:
case POSIX_STATS:
case POSIX_MMAPS:
case POSIX_FOPENS:
case POSIX_FREADS:
case POSIX_FWRITES:
case POSIX_FSEEKS:
case POSIX_FSYNCS:
case POSIX_FDSYNCS:
case POSIX_BYTES_READ:
case POSIX_BYTES_WRITTEN:
case POSIX_CONSEC_READS:
case POSIX_CONSEC_WRITES:
case POSIX_SEQ_READS:
case POSIX_SEQ_WRITES:
case POSIX_RW_SWITCHES:
case POSIX_MEM_NOT_ALIGNED:
case POSIX_FILE_NOT_ALIGNED:
case POSIX_SIZE_READ_0_100:
case POSIX_SIZE_READ_100_1K:
case POSIX_SIZE_READ_1K_10K:
case POSIX_SIZE_READ_10K_100K:
case POSIX_SIZE_READ_100K_1M:
case POSIX_SIZE_READ_1M_4M:
case POSIX_SIZE_READ_4M_10M:
case POSIX_SIZE_READ_10M_100M:
case POSIX_SIZE_READ_100M_1G:
case POSIX_SIZE_READ_1G_PLUS:
case POSIX_SIZE_WRITE_0_100:
case POSIX_SIZE_WRITE_100_1K:
case POSIX_SIZE_WRITE_1K_10K:
case POSIX_SIZE_WRITE_10K_100K:
case POSIX_SIZE_WRITE_100K_1M:
case POSIX_SIZE_WRITE_1M_4M:
case POSIX_SIZE_WRITE_4M_10M:
case POSIX_SIZE_WRITE_10M_100M:
case POSIX_SIZE_WRITE_100M_1G:
/* sum */
agg_psx_rec->counters[i] += psx_rec->counters[i];
break;
case POSIX_MODE:
case POSIX_MEM_ALIGNMENT:
case POSIX_FILE_ALIGNMENT:
/* just set to the input value */
agg_psx_rec->counters[i] = psx_rec->counters[i];
break;
case POSIX_MAX_BYTE_READ:
case POSIX_MAX_BYTE_WRITTEN:
/* max */
if(psx_rec->counters[i] > agg_psx_rec->counters[i])
{
agg_psx_rec->counters[i] = psx_rec->counters[i];
}
break;
case POSIX_MAX_READ_TIME_SIZE:
case POSIX_MAX_WRITE_TIME_SIZE:
case POSIX_FASTEST_RANK:
case POSIX_FASTEST_RANK_BYTES:
case POSIX_SLOWEST_RANK:
case POSIX_SLOWEST_RANK_BYTES:
/* these are set with the FP counters */
break;
default:
/* TODO: common access counters and strides */
agg_psx_rec->counters[i] = -1;
break;
}
}
for(i = 0; i < POSIX_F_NUM_INDICES; i++)
{
switch(i)
{
case POSIX_F_READ_TIME:
case POSIX_F_WRITE_TIME:
case POSIX_F_META_TIME:
/* sum */
agg_psx_rec->fcounters[i] += psx_rec->fcounters[i];
break;
case POSIX_F_OPEN_TIMESTAMP:
case POSIX_F_READ_START_TIMESTAMP:
case POSIX_F_WRITE_START_TIMESTAMP:
/* minimum */
if(psx_rec->fcounters[i] > 0 &&
(psx_rec->fcounters[i] < agg_psx_rec->fcounters[i]))
{
agg_psx_rec->fcounters[i] = psx_rec->fcounters[i];
}
break;
case POSIX_F_READ_END_TIMESTAMP:
case POSIX_F_WRITE_END_TIMESTAMP:
case POSIX_F_CLOSE_TIMESTAMP:
/* maximum */
if(psx_rec->fcounters[i] > agg_psx_rec->fcounters[i])
{
agg_psx_rec->fcounters[i] = psx_rec->fcounters[i];
}
break;
case POSIX_F_MAX_READ_TIME:
if(psx_rec->fcounters[i] > agg_psx_rec->fcounters[i])
{
agg_psx_rec->fcounters[i] = psx_rec->fcounters[i];
agg_psx_rec->counters[POSIX_MAX_READ_TIME_SIZE] =
psx_rec->counters[POSIX_MAX_READ_TIME_SIZE];
}
break;
case POSIX_F_MAX_WRITE_TIME:
if(psx_rec->fcounters[i] > agg_psx_rec->fcounters[i])
{
agg_psx_rec->fcounters[i] = psx_rec->fcounters[i];
agg_psx_rec->counters[POSIX_MAX_WRITE_TIME_SIZE] =
psx_rec->counters[POSIX_MAX_WRITE_TIME_SIZE];
}
break;
case POSIX_F_FASTEST_RANK_TIME:
if(psx_time < agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME])
{
agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] =
psx_rec->counters[POSIX_BYTES_READ] +
psx_rec->counters[POSIX_BYTES_WRITTEN];
agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
}
break;
case POSIX_F_SLOWEST_RANK_TIME:
if(psx_time > agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME])
{
agg_psx_rec->counters[POSIX_SLOWEST_RANK] = psx_rec->base_rec.rank;
agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] =
psx_rec->counters[POSIX_BYTES_READ] +
psx_rec->counters[POSIX_BYTES_WRITTEN];
agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] = psx_time;
}
break;
default:
/* TODO: variance */
agg_psx_rec->fcounters[i] = -1;
break;
}
}
return;
}
/*
* Local variables:
* c-indent-level: 4
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment