Skip to content

GitLab

Commit bb625cbd authored by Shane Snyder's avatar Shane Snyder
Browse files

allow common counters to be updated on the go

parent a22233d7
Hide whitespace changes
Inline Side-by-side
Showing with 96 additions and 43 deletions
darshan-runtime/darshan-common.h
View file @ bb625cbd
......@@ -73,18 +73,24 @@
* base of the value counters (i.e., the first of 4 contiguous common value
* counters) and __cnt_p is a pointer to the base of the count counters (i.e.
* the first of 4 contiguous common count counters). It is assumed your counters
* are stored as int64_t types.
* are stored as int64_t types. __online_flag is set if the common val counters are
* updated during runtime (as opposed to being updated once at darshan shutdown time).
*/
#define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count) do {\
#define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count, __online_flag) do {\
int i; \
int set = 0; \
int64_t min = *(__cnt_p); \
int min_index = 0; \
int inc_count; \
if(__value == 0) break; \
if(__online_flag) \
inc_count = 1; \
else \
inc_count = __count; \
for(i=0; i<4; i++) { \
/* increment bucket if already exists */ \
if(*(__val_p + i) == __value) { \
*(__cnt_p + i) += __count; \
*(__cnt_p + i) += inc_count; \
set = 1; \
break; \
} \
......@@ -138,7 +144,7 @@ struct darshan_variance_dt
* path string.
*/
char* darshan_clean_file_path(
const char* path);
const char *path);
/* darshan_common_val_counter()
*
......@@ -148,13 +154,19 @@ char* darshan_clean_file_path(
* used by a specific module, for instance. 'common_val_root' is the
* root pointer for the tree which stores common value info,
* 'common_val_count' is a pointer to the number of nodes in the
* tree (i.e., the number of allocated common value counters), and
* 'val' is the new value to attempt to add.
* tree (i.e., the number of allocated common value counters), 'val'
* is the new value to attempt to add, 'val_p' is a pointer to the
* base counter (i.e., the first) of the common values (which are
* assumed to be 4 total and contiguous in memory), and 'cnt_p' is
* a pointer to the base counter of the common counts (which are
* again expected to be contiguous in memory).
*/
void darshan_common_val_counter(
void** common_val_root,
int* common_val_count,
int64_t val);
void **common_val_root,
int *common_val_count,
int64_t val,
int64_t *val_p,
int64_t *cnt_p);
/* darshan_walk_common_vals()
*
......@@ -168,9 +180,9 @@ void darshan_common_val_counter(
* (which are again expected to be contiguous in memory).
*/
void darshan_walk_common_vals(
void* common_val_root,
int64_t* val_p,
int64_t* cnt_p);
void *common_val_root,
int64_t *val_p,
int64_t *cnt_p);
/* darshan_variance_reduce()
*
......
darshan-runtime/lib/darshan-common.c
View file @ bb625cbd
......@@ -83,7 +83,7 @@ static int64_t* walker_val_p = NULL;
static int64_t* walker_cnt_p = NULL;
void darshan_common_val_counter(void **common_val_root, int *common_val_count,
int64_t val)
int64_t val, int64_t *common_val_p, int64_t *common_cnt_p)
{
struct darshan_common_val_counter* counter;
struct darshan_common_val_counter* found;
......@@ -102,12 +102,10 @@ void darshan_common_val_counter(void **common_val_root, int *common_val_count,
{
found = *(struct darshan_common_val_counter**)tmp;
found->freq++;
return;
}
/* we can add a new one as long as we haven't hit the limit */
if(*common_val_count < DARSHAN_COMMON_VAL_MAX_RUNTIME_COUNT)
else if(*common_val_count < DARSHAN_COMMON_VAL_MAX_RUNTIME_COUNT)
{
/* we can add a new one as long as we haven't hit the limit */
counter = malloc(sizeof(*counter));
if(!counter)
{
......@@ -127,22 +125,28 @@ void darshan_common_val_counter(void **common_val_root, int *common_val_count,
(*common_val_count)++;
}
#ifdef __DARSHAN_ENABLE_MMAP_LOGS
/* if we are using darshan's mmap feature, update common access
* counters as we go
*/
DARSHAN_COMMON_VAL_COUNTER_INC(common_val_p, common_cnt_p,
found->val, found->freq, 1);
#endif
return;
}
void darshan_walk_common_vals(void *common_val_root, int64_t* val_p,
int64_t* cnt_p)
void darshan_walk_common_vals(void *common_val_root, int64_t *val_p,
int64_t *cnt_p)
{
walker_val_p = val_p;
walker_cnt_p = cnt_p;
twalk(common_val_root, darshan_common_val_walker);
tdestroy(common_val_root, free);
return;
}
static void darshan_common_val_walker(const void* nodep, const VISIT which,
static void darshan_common_val_walker(const void *nodep, const VISIT which,
const int depth)
{
struct darshan_common_val_counter* counter;
......@@ -153,7 +157,7 @@ static void darshan_common_val_walker(const void* nodep, const VISIT which,
case leaf:
counter = *(struct darshan_common_val_counter**)nodep;
DARSHAN_COMMON_VAL_COUNTER_INC(walker_val_p, walker_cnt_p,
counter->val, counter->freq);
counter->val, counter->freq, 0);
default:
break;
}
......@@ -161,7 +165,7 @@ static void darshan_common_val_walker(const void* nodep, const VISIT which,
return;
}
static int darshan_common_val_compare(const void* a_p, const void* b_p)
static int darshan_common_val_compare(const void *a_p, const void *b_p)
{
const struct darshan_common_val_counter* a = a_p;
const struct darshan_common_val_counter* b = b_p;
......
darshan-runtime/lib/darshan-mpiio.c
View file @ bb625cbd
......@@ -4,6 +4,9 @@
*
*/
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
......@@ -17,7 +20,6 @@
#include <errno.h>
#include <search.h>
#include <assert.h>
#define __USE_GNU
#include <pthread.h>
#include "uthash.h"
......@@ -138,7 +140,9 @@ static void mpiio_shutdown(void);
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[MPIIO_SIZE_READ_AGG_0_100]), size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size, \
&(file->file_record->counters[MPIIO_ACCESS1_ACCESS]), \
&(file->file_record->counters[MPIIO_ACCESS1_COUNT])); \
file->file_record->counters[MPIIO_BYTES_READ] += size; \
file->file_record->counters[__counter] += 1; \
if(file->last_io_type == DARSHAN_IO_WRITE) \
......@@ -150,7 +154,8 @@ static void mpiio_shutdown(void);
if(file->file_record->fcounters[MPIIO_F_MAX_READ_TIME] < __elapsed) { \
file->file_record->fcounters[MPIIO_F_MAX_READ_TIME] = __elapsed; \
file->file_record->counters[MPIIO_MAX_READ_TIME_SIZE] = size; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_READ_TIME], __tm1, __tm2, file->last_read_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_READ_TIME], \
__tm1, __tm2, file->last_read_end); \
} while(0)
#define MPIIO_RECORD_WRITE(__ret, __fh, __count, __datatype, __counter, __tm1, __tm2) do { \
......@@ -163,7 +168,9 @@ static void mpiio_shutdown(void);
DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size); \
size = size * __count; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[MPIIO_SIZE_WRITE_AGG_0_100]), size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size); \
darshan_common_val_counter(&file->access_root, &file->access_count, size, \
&(file->file_record->counters[MPIIO_ACCESS1_ACCESS]), \
&(file->file_record->counters[MPIIO_ACCESS1_COUNT])); \
file->file_record->counters[MPIIO_BYTES_WRITTEN] += size; \
file->file_record->counters[__counter] += 1; \
if(file->last_io_type == DARSHAN_IO_READ) \
......@@ -175,7 +182,8 @@ static void mpiio_shutdown(void);
if(file->file_record->fcounters[MPIIO_F_MAX_WRITE_TIME] < __elapsed) { \
file->file_record->fcounters[MPIIO_F_MAX_WRITE_TIME] = __elapsed; \
file->file_record->counters[MPIIO_MAX_WRITE_TIME_SIZE] = size; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_WRITE_TIME], __tm1, __tm2, file->last_write_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[MPIIO_F_WRITE_TIME], \
__tm1, __tm2, file->last_write_end); \
} while(0)
/**********************************************************
......@@ -1088,7 +1096,7 @@ static void mpiio_record_reduction_op(
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[MPIIO_ACCESS1_ACCESS]),
&(tmp_file.counters[MPIIO_ACCESS1_COUNT]), infile->counters[j],
infile->counters[j+4]);
infile->counters[j+4], 0);
}
/* second set */
......@@ -1096,7 +1104,7 @@ static void mpiio_record_reduction_op(
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[MPIIO_ACCESS1_ACCESS]),
&(tmp_file.counters[MPIIO_ACCESS1_COUNT]), inoutfile->counters[j],
inoutfile->counters[j+4]);
inoutfile->counters[j+4], 0);
}
/* min non-zero (if available) value */
......@@ -1328,10 +1336,18 @@ static void mpiio_get_output_data(
{
tmp = &(mpiio_runtime->file_runtime_array[i]);
#ifndef __DARSHAN_ENABLE_MMAP_LOGS
/* walk common counters to get 4 most common -- only if mmap
* feature is disabled (mmap updates counters on the go)
*/
/* common access sizes */
darshan_walk_common_vals(tmp->access_root,
&(tmp->file_record->counters[MPIIO_ACCESS1_ACCESS]),
&(tmp->file_record->counters[MPIIO_ACCESS1_COUNT]));
#endif
tdestroy(tmp->access_root, free);
}
/* if there are globally shared files, do a shared file reduction */
......
darshan-runtime/lib/darshan-posix.c
View file @ bb625cbd
......@@ -220,7 +220,8 @@ static void posix_shutdown(void);
file->file_record->counters[POSIX_OPENS] += 1; \
if(file->file_record->fcounters[POSIX_F_OPEN_TIMESTAMP] == 0) \
file->file_record->fcounters[POSIX_F_OPEN_TIMESTAMP] = __tm1; \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_META_TIME], __tm1, __tm2, file->last_meta_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_META_TIME], \
__tm1, __tm2, file->last_meta_end); \
} while(0)
#define POSIX_RECORD_READ(__ret, __fd, __pread_flag, __pread_offset, __aligned, __stream_flag, __tm1, __tm2) do{ \
......@@ -255,8 +256,12 @@ static void posix_shutdown(void);
else \
file->file_record->counters[POSIX_READS] += 1; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[POSIX_SIZE_READ_0_100]), __ret); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret, \
&(file->file_record->counters[POSIX_ACCESS1_ACCESS]), \
&(file->file_record->counters[POSIX_ACCESS1_COUNT])); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride, \
&(file->file_record->counters[POSIX_STRIDE1_STRIDE]), \
&(file->file_record->counters[POSIX_STRIDE1_COUNT])); \
if(!__aligned) \
file->file_record->counters[POSIX_MEM_NOT_ALIGNED] += 1; \
file_alignment = file->file_record->counters[POSIX_FILE_ALIGNMENT]; \
......@@ -271,7 +276,8 @@ static void posix_shutdown(void);
if(file->file_record->fcounters[POSIX_F_MAX_READ_TIME] < __elapsed) { \
file->file_record->fcounters[POSIX_F_MAX_READ_TIME] = __elapsed; \
file->file_record->counters[POSIX_MAX_READ_TIME_SIZE] = __ret; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_READ_TIME], __tm1, __tm2, file->last_read_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_READ_TIME], \
__tm1, __tm2, file->last_read_end); \
} while(0)
#define POSIX_RECORD_WRITE(__ret, __fd, __pwrite_flag, __pwrite_offset, __aligned, __stream_flag, __tm1, __tm2) do{ \
......@@ -306,8 +312,12 @@ static void posix_shutdown(void);
else \
file->file_record->counters[POSIX_WRITES] += 1; \
DARSHAN_BUCKET_INC(&(file->file_record->counters[POSIX_SIZE_WRITE_0_100]), __ret); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride); \
darshan_common_val_counter(&file->access_root, &file->access_count, __ret, \
&(file->file_record->counters[POSIX_ACCESS1_ACCESS]), \
&(file->file_record->counters[POSIX_ACCESS1_COUNT])); \
darshan_common_val_counter(&file->stride_root, &file->stride_count, stride, \
&(file->file_record->counters[POSIX_STRIDE1_STRIDE]), \
&(file->file_record->counters[POSIX_STRIDE1_COUNT])); \
if(!__aligned) \
file->file_record->counters[POSIX_MEM_NOT_ALIGNED] += 1; \
file_alignment = file->file_record->counters[POSIX_FILE_ALIGNMENT]; \
......@@ -322,7 +332,8 @@ static void posix_shutdown(void);
if(file->file_record->fcounters[POSIX_F_MAX_WRITE_TIME] < __elapsed) { \
file->file_record->fcounters[POSIX_F_MAX_WRITE_TIME] = __elapsed; \
file->file_record->counters[POSIX_MAX_WRITE_TIME_SIZE] = __ret; } \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_WRITE_TIME], __tm1, __tm2, file->last_write_end); \
DARSHAN_TIMER_INC_NO_OVERLAP(file->file_record->fcounters[POSIX_F_WRITE_TIME], \
__tm1, __tm2, file->last_write_end); \
} while(0)
#define POSIX_LOOKUP_RECORD_STAT(__path, __statbuf, __tm1, __tm2) do { \
......@@ -336,8 +347,9 @@ static void posix_shutdown(void);
} while(0)
#define POSIX_RECORD_STAT(__file, __statbuf, __tm1, __tm2) do { \
DARSHAN_TIMER_INC_NO_OVERLAP((__file)->file_record->fcounters[POSIX_F_META_TIME], __tm1, __tm2, (__file)->last_meta_end); \
(__file)->file_record->counters[POSIX_STATS] += 1; \
DARSHAN_TIMER_INC_NO_OVERLAP((__file)->file_record->fcounters[POSIX_F_META_TIME], \
__tm1, __tm2, (__file)->last_meta_end); \
} while(0)
......@@ -1773,14 +1785,14 @@ static void posix_record_reduction_op(void* infile_v, void* inoutfile_v,
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_STRIDE1_STRIDE]),
&(tmp_file.counters[POSIX_STRIDE1_COUNT]), infile->counters[j],
infile->counters[j+4]);
infile->counters[j+4], 0);
}
/* second set */
for(j=POSIX_STRIDE1_STRIDE; j<=POSIX_STRIDE4_STRIDE; j++)
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_STRIDE1_STRIDE]),
&(tmp_file.counters[POSIX_STRIDE1_COUNT]), inoutfile->counters[j],
inoutfile->counters[j+4]);
inoutfile->counters[j+4], 0);
}
/* same for access counts */
......@@ -1804,14 +1816,14 @@ static void posix_record_reduction_op(void* infile_v, void* inoutfile_v,
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_ACCESS1_ACCESS]),
&(tmp_file.counters[POSIX_ACCESS1_COUNT]), infile->counters[j],
infile->counters[j+4]);
infile->counters[j+4], 0);
}
/* second set */
for(j=POSIX_ACCESS1_ACCESS; j<=POSIX_ACCESS4_ACCESS; j++)
{
DARSHAN_COMMON_VAL_COUNTER_INC(&(tmp_file.counters[POSIX_ACCESS1_ACCESS]),
&(tmp_file.counters[POSIX_ACCESS1_COUNT]), inoutfile->counters[j],
inoutfile->counters[j+4]);
inoutfile->counters[j+4], 0);
}
/* min non-zero (if available) value */
......@@ -2046,6 +2058,11 @@ static void posix_get_output_data(
{
tmp = &(posix_runtime->file_runtime_array[i]);
#ifndef __DARSHAN_ENABLE_MMAP_LOGS
/* walk common counters to get 4 most common -- only if mmap
* feature is disabled (mmap updates counters on the go)
*/
/* common accesses */
darshan_walk_common_vals(tmp->access_root,
&(tmp->file_record->counters[POSIX_ACCESS1_ACCESS]),
......@@ -2054,6 +2071,10 @@ static void posix_get_output_data(
darshan_walk_common_vals(tmp->stride_root,
&(tmp->file_record->counters[POSIX_STRIDE1_STRIDE]),
&(tmp->file_record->counters[POSIX_STRIDE1_COUNT]));
#endif
tdestroy(tmp->access_root, free);
tdestroy(tmp->stride_root, free);
}
/* if there are globally shared files, do a shared file reduction */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment