Skip to content

D
darshan
Commit bb625cbd authored by Shane Snyder's avatar Shane Snyder
Browse files
Options

allow common counters to be updated on the go

parent a22233d7
Hide whitespace changes
Inline Side-by-side
Showing with 96 additions and 43 deletions
darshan-runtime/darshan-common.h
View file @ bb625cbd
......@@ -73,18 +73,24 @@
* base of the value counters (i.e., the first of 4 contiguous common value
* counters) and __cnt_p is a pointer to the base of the count counters (i.e.
* the first of 4 contiguous common count counters). It is assumed your counters
* are stored as int64_t types.
* are stored as int64_t types. __online_flag is set if the common val counters are
* updated during runtime (as opposed to being updated once at darshan shutdown time).
*/
#define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count) do {\
#define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count, __online_flag) do {\
int i; \
int set = 0; \
int64_t min = *(__cnt_p); \
int min_index = 0; \
int inc_count; \
if(__value == 0) break; \
if(__online_flag) \
inc_count = 1; \
else \
inc_count = __count; \
for(i=0; i<4; i++) { \
/* increment bucket if already exists */ \
if(*(__val_p + i) == __value) { \
*(__cnt_p + i) += __count; \
*(__cnt_p + i) += inc_count; \
set = 1; \
break; \
} \
......@@ -138,7 +144,7 @@ struct darshan_variance_dt
* path string.
*/
char* darshan_clean_file_path(
const char* path);
const char *path);
/* darshan_common_val_counter()
*
......@@ -148,13 +154,19 @@ char* darshan_clean_file_path(
* used by a specific module, for instance. 'common_val_root' is the
* root pointer for the tree which stores common value info,
* 'common_val_count' is a pointer to the number of nodes in the
* tree (i.e., the number of allocated common value counters), and
* 'val' is the new value to attempt to add.
* tree (i.e., the number of allocated common value counters), 'val'
* is the new value to attempt to add, 'val_p' is a pointer to the
* base counter (i.e., the first) of the common values (which are
* assumed to be 4 total and contiguous in memory), and 'cnt_p' is
* a pointer to the base counter of the common counts (which are
* again expected to be contiguous in memory).
*/
void darshan_common_val_counter(
void** common_val_root,
int* common_val_count,
int64_t val);
void **common_val_root,
int *common_val_count,
int64_t val,
int64_t *val_p,
int64_t *cnt_p);
/* darshan_walk_common_vals()
*
......@@ -168,9 +180,9 @@ void darshan_common_val_counter(
* (which are again expected to be contiguous in memory).
*/
void darshan_walk_common_vals(
void* common_val_root,
int64_t* val_p,
int64_t* cnt_p);
void *common_val_root,
int64_t *val_p,
int64_t *cnt_p);
/* darshan_variance_reduce()
*
......
darshan-runtime/lib/darshan-common.c
View file @ bb625cbd
......@@ -83,7 +83,7 @@ static int64_t* walker_val_p = NULL;
static int64_t* walker_cnt_p = NULL;
void darshan_common_val_counter(void **common_val_root, int *common_val_count,
int64_t val)
int64_t val, int64_t *common_val_p, int64_t *common_cnt_p)
{
struct darshan_common_val_counter* counter;
struct darshan_common_val_counter* found;
......@@ -102,12 +102,10 @@ void darshan_common_val_counter(void **common_val_root, int *common_val_count,
{
found = *(struct darshan_common_val_counter**)tmp;
found->freq++;
return;
}
/* we can add a new one as long as we haven't hit the limit */
if(*common_val_count < DARSHAN_COMMON_VAL_MAX_RUNTIME_COUNT)
else if(*common_val_count < DARSHAN_COMMON_VAL_MAX_RUNTIME_COUNT)
{
/* we can add a new one as long as we haven't hit the limit */
counter = malloc(sizeof(*counter));
if(!counter)
{
......@@ -127,22 +125,28 @@ void darshan_common_val_counter(void **common_val_root, int *common_val_count,
(*common_val_count)++;
}
#ifdef __DARSHAN_ENABLE_MMAP_LOGS
/* if we are using darshan's mmap feature, update common access
* counters as we go
*/
DARSHAN_COMMON_VAL_COUNTER_INC(common_val_p, common_cnt_p,
found->val, found->freq, 1);
#endif
return;
}
void darshan_walk_common_vals(void *common_val_root, int64_t* val_p,
int64_t* cnt_p)
void darshan_walk_common_vals(void *common_val_root, int64_t *val_p,
int64_t *cnt_p)
{
walker_val_p = val_p;
walker_cnt_p = cnt_p;
twalk(common_val_root, darshan_common_val_walker);
tdestroy(common_val_root, free);
return;
}
static void darshan_common_val_walker(const void* nodep, const VISIT which,
static void darshan_common_val_walker(const void *nodep, const VISIT which,
const int depth)
{
struct darshan_common_val_counter* counter;
......@@ -153,7 +157,7 @@ static void darshan_common_val_walker(const void* nodep, const VISIT which,
case leaf:
counter = *(struct darshan_common_val_counter**)nodep;
DARSHAN_COMMON_VAL_COUNTER_INC(walker_val_p, walker_cnt_p,
counter->val, counter->freq);
counter->val, counter->freq, 0);
default:
break;
}
......@@ -161,7 +165,7 @@ static void darshan_common_val_walker(const void* nodep, const VISIT which,
return;
}
static int darshan_common_val_compare(const void* a_p, const void* b_p)
static int darshan_common_val_compare(const void *a_p, const void *b_p)
{
const struct darshan_common_val_counter* a = a_p;
const struct darshan_common_val_counter* b = b_p;
......
darshan-runtime/lib/darshan-mpiio.c
View file @ bb625cbd
......@@ -4,6 +4,9 @@
*
*/
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
......@@ -17,7 +20,6 @@
#include <errno.h>
#include <search.h>
#include <assert.h>
#define __USE_GNU
#include <pthread.h>
#include "uthash.h"
......@@ -138,7 +140,9 @@ static void mpiio_shutdown(void);
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[MPIIO_SIZE_READ_AGG_0_100]), size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size, \
&(file->file_record->counters[MPIIO_ACCESS1_ACCESS]), \
&(file->file_record->counters[MPIIO_ACCESS1_COUNT])); \
file->file_record->counters[MPIIO_BYTES_READ] += size; \
file->file_record->counters[__counter] += 1; \
if(file->last_io_type == DARSHAN_IO_WRITE) \
......@@ -150,7 +154,8 @@ static void mpiio_shutdown(void);
if(file->file_record->fcounters[MPIIO_F_MAX_READ_TIME] < __elapsed) { \
file->file_record->fcounters[MPIIO_F_MAX_READ_TIME] = __elapsed; \
file->file_record->counters[MPIIO_MAX_READ_TIME_SIZE] = size; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_READ_TIME], __tm1, __tm2, file->last_read_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_READ_TIME], \
__tm1, __tm2, file->last_read_end); \
} while(0)
#define MPIIO_RECORD_WRITE(__ret, __fh, __count, __datatype, __counter, __tm1, __tm2) do { \
......@@ -163,7 +168,9 @@ static void mpiio_shutdown(void);
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[MPIIO_SIZE_WRITE_AGG_0_100]), size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size, \
&(file->file_record->counters[MPIIO_ACCESS1_ACCESS]), \
&(file->file_record->counters[MPIIO_ACCESS1_COUNT])); \
file->file_record->counters[MPIIO_BYTES_WRITTEN] += size; \
file->file_record->counters[__counter] += 1; \
if(file->last_io_type == DARSHAN_IO_READ) \
......@@ -175,7 +182,8 @@ static void mpiio_shutdown(void);
if(file->file_record->fcounters[MPIIO_F_MAX_WRITE_TIME] < __elapsed) { \
file->file_record->fcounters[MPIIO_F_MAX_WRITE_TIME] = __elapsed; \
file->file_record->counters[MPIIO_MAX_WRITE_TIME_SIZE] = size; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_WRITE_TIME], __tm1, __tm2, file->last_write_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_WRITE_TIME], \
__tm1, __tm2, file->last_write_end); \
} while(0)
/**********************************************************
......@@ -1088,7 +1096,7 @@ static void mpiio_record_reduction_op(
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[MPIIO_ACCESS1_ACCESS]),
&(tmp_file.counters[MPIIO_ACCESS1_COUNT]), infile->counters[j],
infile->counters[j+4]);
infile->counters[j+4], 0);
}
/* second set */
......@@ -1096,7 +1104,7 @@ static void mpiio_record_reduction_op(
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[MPIIO_ACCESS1_ACCESS]),
&(tmp_file.counters[MPIIO_ACCESS1_COUNT]), inoutfile->counters[j],
inoutfile->counters[j+4]);
inoutfile->counters[j+4], 0);
}
/* min non-zero (if available) value */
......@@ -1328,10 +1336,18 @@ static void mpiio_get_output_data(
{
tmp = &(mpiio_runtime->file_runtime_array[i]);
#ifndef __DARSHAN_ENABLE_MMAP_LOGS
/* walk common counters to get 4 most common -- only if mmap
* feature is disabled (mmap updates counters on the go)
*/
/* common access sizes */
darshan_walk_common_vals(tmp->access_root,
&(tmp->file_record->counters[MPIIO_ACCESS1_ACCESS]),
&(tmp->file_record->counters[MPIIO_ACCESS1_COUNT]));
#endif
tdestroy(tmp->access_root, free);
}
/* if there are globally shared files, do a shared file reduction */
......
darshan-runtime/lib/darshan-posix.c
View file @ bb625cbd
......@@ -220,7 +220,8 @@ static void posix_shutdown(void);
file->file_record->counters[POSIX_OPENS] += 1; \
if(file->file_record->fcounters[POSIX_F_OPEN_TIMESTAMP] == 0) \
file->file_record->fcounters[POSIX_F_OPEN_TIMESTAMP] = __tm1; \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_META_TIME], __tm1, __tm2, file->last_meta_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_META_TIME], \
__tm1, __tm2, file->last_meta_end); \
} while(0)
#define POSIX_RECORD_READ(__ret, __fd, __pread_flag, __pread_offset, __aligned, __stream_flag, __tm1, __tm2) do{ \
......@@ -255,8 +256,12 @@ static void posix_shutdown(void);
else \
file->file_record->counters[POSIX_READS] += 1; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[POSIX_SIZE_READ_0_100]), __ret); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret, \
&(file->file_record->counters[POSIX_ACCESS1_ACCESS]), \
&(file->file_record->counters[POSIX_ACCESS1_COUNT])); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride, \
&(file->file_record->counters[POSIX_STRIDE1_STRIDE]), \
&(file->file_record->counters[POSIX_STRIDE1_COUNT])); \
if(!__aligned) \
file->file_record->counters[POSIX_MEM_NOT_ALIGNED] += 1; \
file_alignment = file->file_record->counters[POSIX_FILE_ALIGNMENT]; \
......@@ -271,7 +276,8 @@ static void posix_shutdown(void);
if(file->file_record->fcounters[POSIX_F_MAX_READ_TIME] < __elapsed) { \
file->file_record->fcounters[POSIX_F_MAX_READ_TIME] = __elapsed; \
file->file_record->counters[POSIX_MAX_READ_TIME_SIZE] = __ret; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_READ_TIME], __tm1, __tm2, file->last_read_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_READ_TIME], \
__tm1, __tm2, file->last_read_end); \
} while(0)
#define POSIX_RECORD_WRITE(__ret, __fd, __pwrite_flag, __pwrite_offset, __aligned, __stream_flag, __tm1, __tm2) do{ \
......@@ -306,8 +312,12 @@ static void posix_shutdown(void);
else \
file->file_record->counters[POSIX_WRITES] += 1; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[POSIX_SIZE_WRITE_0_100]), __ret); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret, \
&(file->file_record->counters[POSIX_ACCESS1_ACCESS]), \
&(file->file_record->counters[POSIX_ACCESS1_COUNT])); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride, \
&(file->file_record->counters[POSIX_STRIDE1_STRIDE]), \
&(file->file_record->counters[POSIX_STRIDE1_COUNT])); \
if(!__aligned) \
file->file_record->counters[POSIX_MEM_NOT_ALIGNED] += 1; \
file_alignment = file->file_record->counters[POSIX_FILE_ALIGNMENT]; \
......@@ -322,7 +332,8 @@ static void posix_shutdown(void);
if(file->file_record->fcounters[POSIX_F_MAX_WRITE_TIME] < __elapsed) { \
file->file_record->fcounters[POSIX_F_MAX_WRITE_TIME] = __elapsed; \
file->file_record->counters[POSIX_MAX_WRITE_TIME_SIZE] = __ret; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_WRITE_TIME], __tm1, __tm2, file->last_write_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_WRITE_TIME], \
__tm1, __tm2, file->last_write_end); \
} while(0)
#define POSIX_LOOKUP_RECORD_STAT(__path, __statbuf, __tm1, __tm2) do { \
......@@ -336,8 +347,9 @@ static void posix_shutdown(void);
} while(0)
#define POSIX_RECORD_STAT(__file, __statbuf, __tm1, __tm2) do { \
DARSHAN_TIMER_INC_NO_OVERLAP((__file)->file_record->fcounters[POSIX_F_META_TIME], __tm1, __tm2, (__file)->last_meta_end); \
(__file)->file_record->counters[POSIX_STATS] += 1; \
DARSHAN_TIMER_INC_NO_OVERLAP((__file)->file_record->fcounters[POSIX_F_META_TIME], \
__tm1, __tm2, (__file)->last_meta_end); \
} while(0)
......@@ -1773,14 +1785,14 @@ static void posix_record_reduction_op(void* infile_v, void* inoutfile_v,
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_STRIDE1_STRIDE]),
&(tmp_file.counters[POSIX_STRIDE1_COUNT]), infile->counters[j],
infile->counters[j+4]);
infile->counters[j+4], 0);
}
/* second set */
for(j=POSIX_STRIDE1_STRIDE; j<=POSIX_STRIDE4_STRIDE; j++)
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_STRIDE1_STRIDE]),
&(tmp_file.counters[POSIX_STRIDE1_COUNT]), inoutfile->counters[j],
inoutfile->counters[j+4]);
inoutfile->counters[j+4], 0);
}
/* same for access counts */
......@@ -1804,14 +1816,14 @@ static void posix_record_reduction_op(void* infile_v, void* inoutfile_v,
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_ACCESS1_ACCESS]),
&(tmp_file.counters[POSIX_ACCESS1_COUNT]), infile->counters[j],
infile->counters[j+4]);
infile->counters[j+4], 0);
}
/* second set */
for(j=POSIX_ACCESS1_ACCESS; j<=POSIX_ACCESS4_ACCESS; j++)
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_ACCESS1_ACCESS]),
&(tmp_file.counters[POSIX_ACCESS1_COUNT]), inoutfile->counters[j],
inoutfile->counters[j+4]);
inoutfile->counters[j+4], 0);
}
/* min non-zero (if available) value */
......@@ -2046,6 +2058,11 @@ static void posix_get_output_data(
{
tmp = &(posix_runtime->file_runtime_array[i]);
#ifndef __DARSHAN_ENABLE_MMAP_LOGS
/* walk common counters to get 4 most common -- only if mmap
* feature is disabled (mmap updates counters on the go)
*/
/* common accesses */
darshan_walk_common_vals(tmp->access_root,
&(tmp->file_record->counters[POSIX_ACCESS1_ACCESS]),
......@@ -2054,6 +2071,10 @@ static void posix_get_output_data(
darshan_walk_common_vals(tmp->stride_root,
&(tmp->file_record->counters[POSIX_STRIDE1_STRIDE]),
&(tmp->file_record->counters[POSIX_STRIDE1_COUNT]));
#endif
tdestroy(tmp->access_root, free);
tdestroy(tmp->stride_root, free);
}
/* if there are globally shared files, do a shared file reduction */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment