Commit 158337c9 authored by Shane Snyder's avatar Shane Snyder
Browse files

add code for the pnetcdf module

parent b6807fcc
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef __DARSHAN_PNETCDF_LOG_FORMAT_H
#define __DARSHAN_PNETCDF_LOG_FORMAT_H
#include "darshan-log-format.h"
#define PNETCDF_COUNTERS \
/* count of PNETCDF independent opens */\
X(PNETCDF_INDEP_OPENS) \
/* count of PNETCDF collective opens */\
X(PNETCDF_COLL_OPENS) \
/* end of counters */\
X(PNETCDF_NUM_INDICES)
#define PNETCDF_F_COUNTERS \
/* timestamp of first open */\
X(PNETCDF_F_OPEN_TIMESTAMP) \
/* timestamp of last close */\
X(PNETCDF_F_CLOSE_TIMESTAMP) \
/* end of counters*/\
X(PNETCDF_F_NUM_INDICES)
#define X(a) a,
/* integer statistics for PNETCDF file records */
enum darshan_pnetcdf_indices
{
PNETCDF_COUNTERS
};
/* floating point statistics for PNETCDF file records */
enum darshan_pnetcdf_f_indices
{
PNETCDF_F_COUNTERS
};
#undef X
/* file record structure for PNETCDF files. a record is created and stored for
* every PNETCDF file opened by the original application. For the PNETCDF module,
* the record includes:
* - a corresponding record identifier (created by hashing the file path)
* - the rank of the process which opened the file (-1 for shared files)
* - integer file I/O statistics (open, read/write counts, etc)
* - floating point I/O statistics (timestamps, cumulative timers, etc.)
*/
struct darshan_pnetcdf_file
{
darshan_record_id f_id;
int64_t rank;
int64_t counters[PNETCDF_NUM_INDICES];
double fcounters[PNETCDF_F_NUM_INDICES];
};
#endif /* __DARSHAN_PNETCDF_LOG_FORMAT_H */
......@@ -75,6 +75,12 @@ lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)
lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/lookup3.o: lib/lookup3.c
$(CC) $(CFLAGS) -c $< -o $@
......@@ -87,13 +93,10 @@ lib/lookup8.o: lib/lookup8.c
lib/lookup8.po: lib/lookup8.c
$(CC) $(CFLAGS_SHARED) -c $< -o $@
#%.i: %.c
# $(CC) -E $(CFLAGS) -c $< -o $@
lib/libdarshan.a: lib/darshan-core-init-finalize.o lib/darshan-core.o lib/darshan-common.o lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/lookup3.o lib/lookup8.o
lib/libdarshan.a: lib/darshan-core-init-finalize.o lib/darshan-core.o lib/darshan-common.o lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o lib/lookup3.o lib/lookup8.o
ar rcs $@ $^
lib/libdarshan.so: lib/darshan-core-init-finalize.po lib/darshan-core.po lib/darshan-common.po lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/lookup3.po lib/lookup8.po
lib/libdarshan.so: lib/darshan-core-init-finalize.po lib/darshan-core.po lib/darshan-common.po lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po lib/lookup3.po lib/lookup8.po
$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ -lpthread -lrt -lz -ldl
install:: all
......
......@@ -497,7 +497,7 @@ static void hdf5_get_output_data(
/* make *send_buf point to the shared files at the end of sorted array */
red_send_buf =
&(hdf5_runtime->file_record_array[hdf5_runtime->file_array_ndx-(shared_rec_count)]);
&(hdf5_runtime->file_record_array[hdf5_runtime->file_array_ndx-shared_rec_count]);
/* allocate memory for the reduction output on rank 0 */
if(my_rank == 0)
......
......@@ -1369,7 +1369,7 @@ static void mpiio_get_output_data(
/* make *send_buf point to the shared files at the end of sorted array */
red_send_buf =
&(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx-(shared_rec_count)]);
&(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx-shared_rec_count]);
/* allocate memory for the reduction output on rank 0 */
if(my_rank == 0)
......
/*
* Copyright (C) 2015 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdarg.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <errno.h>
#include <search.h>
#include <assert.h>
#define __USE_GNU
#include <pthread.h>
#include "uthash.h"
#include "darshan.h"
#include "darshan-pnetcdf-log-format.h"
#include "darshan-dynamic.h"
DARSHAN_FORWARD_DECL(ncmpi_create, int, (MPI_Comm comm, const char *path, int cmode, MPI_Info info, int *ncidp));
DARSHAN_FORWARD_DECL(ncmpi_open, int, (MPI_Comm comm, const char *path, int omode, MPI_Info info, int *ncidp));
DARSHAN_FORWARD_DECL(ncmpi_close, int, (int ncid));
/* structure to track i/o stats for a given PNETCDF file at runtime */
struct pnetcdf_file_runtime
{
struct darshan_pnetcdf_file* file_record;
UT_hash_handle hlink;
};
/* structure to associate a PNETCDF ncid with an existing file runtime structure */
struct pnetcdf_file_runtime_ref
{
struct pnetcdf_file_runtime* file;
int ncid;
UT_hash_handle hlink;
};
/* necessary state for storing PNETCDF file records and coordinating with
* darshan-core at shutdown time
*/
struct pnetcdf_runtime
{
struct pnetcdf_file_runtime* file_runtime_array;
struct darshan_pnetcdf_file* file_record_array;
int file_array_size;
int file_array_ndx;
struct pnetcdf_file_runtime *file_hash;
struct pnetcdf_file_runtime_ref* ncid_hash;
};
static struct pnetcdf_runtime *pnetcdf_runtime = NULL;
static pthread_mutex_t pnetcdf_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int instrumentation_disabled = 0;
static int my_rank = -1;
static void pnetcdf_runtime_initialize(void);
static struct pnetcdf_file_runtime* pnetcdf_file_by_name(const char *name);
static struct pnetcdf_file_runtime* pnetcdf_file_by_name_setncid(const char* name, int ncid);
static struct pnetcdf_file_runtime* pnetcdf_file_by_ncid(int ncid);
static void pnetcdf_file_close_ncid(int ncid);
static int pnetcdf_record_compare(const void* a, const void* b);
static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
int *len, MPI_Datatype *datatype);
static void pnetcdf_begin_shutdown(void);
static void pnetcdf_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **pnetcdf_buf, int *pnetcdf_buf_sz);
static void pnetcdf_shutdown(void);
#define PNETCDF_LOCK() pthread_mutex_lock(&pnetcdf_runtime_mutex)
#define PNETCDF_UNLOCK() pthread_mutex_unlock(&pnetcdf_runtime_mutex)
/*********************************************************
* Wrappers for PNETCDF functions of interest *
*********************************************************/
int DARSHAN_DECL(ncmpi_create)(MPI_Comm comm, const char *path,
int cmode, MPI_Info info, int *ncidp)
{
int ret;
struct pnetcdf_file_runtime* file;
char* tmp;
int comm_size;
double tm1;
MAP_OR_FAIL(ncmpi_create);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_create(comm, path, cmode, info, ncidp);
if(ret == 0)
{
/* use ROMIO approach to strip prefix if present */
/* strip off prefix if there is one, but only skip prefixes
* if they are greater than length one to allow for windows
* drive specifications (e.g. c:\...)
*/
tmp = strchr(path, ':');
if (tmp > path + 1) {
path = tmp + 1;
}
PNETCDF_LOCK();
pnetcdf_runtime_initialize();
file = pnetcdf_file_by_name_setncid(path, (*ncidp));
if(file)
{
if(file->file_record->fcounters[PNETCDF_F_OPEN_TIMESTAMP] == 0)
file->file_record->fcounters[PNETCDF_F_OPEN_TIMESTAMP] = tm1;
DARSHAN_MPI_CALL(PMPI_Comm_size)(comm, &comm_size);
if(comm_size == 1)
{
file->file_record->counters[PNETCDF_INDEP_OPENS] += 1;
}
else
{
file->file_record->counters[PNETCDF_COLL_OPENS] += 1;
}
}
PNETCDF_UNLOCK();
}
return(ret);
}
int DARSHAN_DECL(ncmpi_open)(MPI_Comm comm, const char *path,
int omode, MPI_Info info, int *ncidp)
{
int ret;
struct pnetcdf_file_runtime* file;
char* tmp;
int comm_size;
double tm1;
MAP_OR_FAIL(ncmpi_open);
tm1 = darshan_core_wtime();
ret = __real_ncmpi_open(comm, path, omode, info, ncidp);
if(ret == 0)
{
/* use ROMIO approach to strip prefix if present */
/* strip off prefix if there is one, but only skip prefixes
* if they are greater than length one to allow for windows
* drive specifications (e.g. c:\...)
*/
tmp = strchr(path, ':');
if (tmp > path + 1) {
path = tmp + 1;
}
PNETCDF_LOCK();
pnetcdf_runtime_initialize();
file = pnetcdf_file_by_name_setncid(path, (*ncidp));
if(file)
{
if(file->file_record->fcounters[PNETCDF_F_OPEN_TIMESTAMP] == 0)
file->file_record->fcounters[PNETCDF_F_OPEN_TIMESTAMP] = tm1;
DARSHAN_MPI_CALL(PMPI_Comm_size)(comm, &comm_size);
if(comm_size == 1)
{
file->file_record->counters[PNETCDF_INDEP_OPENS] += 1;
}
else
{
file->file_record->counters[PNETCDF_COLL_OPENS] += 1;
}
}
PNETCDF_UNLOCK();
}
return(ret);
}
int DARSHAN_DECL(ncmpi_close)(int ncid)
{
struct pnetcdf_file_runtime* file;
int ret;
MAP_OR_FAIL(ncmpi_close);
ret = __real_ncmpi_close(ncid);
PNETCDF_LOCK();
pnetcdf_runtime_initialize();
file = pnetcdf_file_by_ncid(ncid);
if(file)
{
file->file_record->fcounters[PNETCDF_F_CLOSE_TIMESTAMP] =
darshan_core_wtime();
pnetcdf_file_close_ncid(ncid);
}
PNETCDF_UNLOCK();
return(ret);
}
/************************************************************
* Internal functions for manipulating PNETCDF module state *
************************************************************/
/* initialize internal PNETCDF module data strucutres and register with darshan-core */
static void pnetcdf_runtime_initialize()
{
int mem_limit;
struct darshan_module_funcs pnetcdf_mod_fns =
{
.begin_shutdown = &pnetcdf_begin_shutdown,
.get_output_data = &pnetcdf_get_output_data,
.shutdown = &pnetcdf_shutdown
};
/* don't do anything if already initialized or instrumenation is disabled */
if(pnetcdf_runtime || instrumentation_disabled)
return;
/* register pnetcdf module with darshan-core */
darshan_core_register_module(
DARSHAN_PNETCDF_MOD,
&pnetcdf_mod_fns,
&my_rank,
&mem_limit,
NULL);
/* return if no memory assigned by darshan-core */
if(mem_limit == 0)
return;
pnetcdf_runtime = malloc(sizeof(*pnetcdf_runtime));
if(!pnetcdf_runtime)
return;
memset(pnetcdf_runtime, 0, sizeof(*pnetcdf_runtime));
/* set maximum number of file records according to max memory limit */
/* NOTE: maximum number of records is based on the size of a pnetcdf file record */
/* TODO: should we base memory usage off file record or total runtime structure sizes? */
pnetcdf_runtime->file_array_size = mem_limit / sizeof(struct darshan_pnetcdf_file);
pnetcdf_runtime->file_array_ndx = 0;
/* allocate array of runtime file records */
pnetcdf_runtime->file_runtime_array = malloc(pnetcdf_runtime->file_array_size *
sizeof(struct pnetcdf_file_runtime));
pnetcdf_runtime->file_record_array = malloc(pnetcdf_runtime->file_array_size *
sizeof(struct darshan_pnetcdf_file));
if(!pnetcdf_runtime->file_runtime_array || !pnetcdf_runtime->file_record_array)
{
pnetcdf_runtime->file_array_size = 0;
return;
}
memset(pnetcdf_runtime->file_runtime_array, 0, pnetcdf_runtime->file_array_size *
sizeof(struct pnetcdf_file_runtime));
memset(pnetcdf_runtime->file_record_array, 0, pnetcdf_runtime->file_array_size *
sizeof(struct darshan_pnetcdf_file));
return;
}
/* get a PNETCDF file record for the given file path */
static struct pnetcdf_file_runtime* pnetcdf_file_by_name(const char *name)
{
struct pnetcdf_file_runtime *file = NULL;
char *newname = NULL;
darshan_record_id file_id;
if(!pnetcdf_runtime || instrumentation_disabled)
return(NULL);
newname = darshan_clean_file_path(name);
if(!newname)
newname = (char*)name;
/* get a unique id for this file from darshan core */
darshan_core_register_record(
(void*)newname,
strlen(newname),
1,
DARSHAN_PNETCDF_MOD,
&file_id,
NULL);
/* search the hash table for this file record, and return if found */
HASH_FIND(hlink, pnetcdf_runtime->file_hash, &file_id, sizeof(darshan_record_id), file);
if(file)
{
if(newname != name)
free(newname);
return(file);
}
if(pnetcdf_runtime->file_array_ndx < pnetcdf_runtime->file_array_size);
{
/* no existing record, assign a new file record from the global array */
file = &(pnetcdf_runtime->file_runtime_array[pnetcdf_runtime->file_array_ndx]);
file->file_record = &(pnetcdf_runtime->file_record_array[pnetcdf_runtime->file_array_ndx]);
file->file_record->f_id = file_id;
file->file_record->rank = my_rank;
/* add new record to file hash table */
HASH_ADD(hlink, pnetcdf_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
pnetcdf_runtime->file_array_ndx++;
}
if(newname != name)
free(newname);
return(file);
}
/* get a PNETCDF file record for the given file path, and also create a
* reference structure using the returned ncid
*/
static struct pnetcdf_file_runtime* pnetcdf_file_by_name_setncid(const char* name, int ncid)
{
struct pnetcdf_file_runtime* file;
struct pnetcdf_file_runtime_ref* ref;
if(!pnetcdf_runtime || instrumentation_disabled)
return(NULL);
/* find file record by name first */
file = pnetcdf_file_by_name(name);
if(!file)
return(NULL);
/* search hash table for existing file ref for this ncid */
HASH_FIND(hlink, pnetcdf_runtime->ncid_hash, &ncid, sizeof(int), ref);
if(ref)
{
/* we have a reference. Make sure it points to the correct file
* and return it
*/
ref->file = file;
return(file);
}
/* if we hit this point, then we don't have a reference for this ncid
* in the table yet. Add it.
*/
ref = malloc(sizeof(*ref));
if(!ref)
return(NULL);
memset(ref, 0, sizeof(*ref));
ref->file = file;
ref->ncid = ncid;
HASH_ADD(hlink, pnetcdf_runtime->ncid_hash, ncid, sizeof(int), ref);
return(file);
}
/* get a PNETCDF file record for the given ncid */
static struct pnetcdf_file_runtime* pnetcdf_file_by_ncid(int ncid)
{
struct pnetcdf_file_runtime_ref* ref;
if(!pnetcdf_runtime || instrumentation_disabled)
return(NULL);
/* search hash table for existing file ref for this ncid */
HASH_FIND(hlink, pnetcdf_runtime->ncid_hash, &ncid, sizeof(int), ref);
if(ref)
return(ref->file);
return(NULL);
}
/* free up PNETCDF reference data structures for the given ncid */
static void pnetcdf_file_close_ncid(int ncid)
{
struct pnetcdf_file_runtime_ref* ref;
if(!pnetcdf_runtime || instrumentation_disabled)
return;
/* search hash table for this ncid */
HASH_FIND(hlink, pnetcdf_runtime->ncid_hash, &ncid, sizeof(int), ref);
if(ref)
{
/* we have a reference, delete it */
HASH_DELETE(hlink, pnetcdf_runtime->ncid_hash, ref);
free(ref);
}
return;
}
/* compare function for sorting file records by descending rank */
static int pnetcdf_record_compare(const void* a_p, const void* b_p)
{
const struct darshan_pnetcdf_file* a = a_p;
const struct darshan_pnetcdf_file* b = b_p;
if(a->rank < b->rank)
return 1;
if(a->rank > b->rank)
return -1;
return 0;
}
static void pnetcdf_record_reduction_op(void* infile_v, void* inoutfile_v,
int *len, MPI_Datatype *datatype)
{
struct darshan_pnetcdf_file tmp_file;
struct darshan_pnetcdf_file *infile = infile_v;
struct darshan_pnetcdf_file *inoutfile = inoutfile_v;
int i, j;
assert(pnetcdf_runtime);
for(i=0; i<*len; i++)
{
memset(&tmp_file, 0, sizeof(struct darshan_pnetcdf_file));
tmp_file.f_id = infile->f_id;
tmp_file.rank = -1;
/* sum */
for(j=PNETCDF_INDEP_OPENS; j<=PNETCDF_COLL_OPENS; j++)
{
tmp_file.counters[j] = infile->counters[j] + inoutfile->counters[j];
}
/* min non-zero (if available) value */
for(j=PNETCDF_F_OPEN_TIMESTAMP; j<=PNETCDF_F_OPEN_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j] && inoutfile->fcounters[j] > 0)
tmp_file.fcounters[j] = inoutfile->fcounters[j];
else
tmp_file.fcounters[j] = infile->fcounters[j];
}
/* max */
for(j=PNETCDF_F_CLOSE_TIMESTAMP; j<=PNETCDF_F_CLOSE_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
else
tmp_file.fcounters[j] = inoutfile->fcounters[j];
}
/* update pointers */
*inoutfile = tmp_file;
inoutfile++;
infile++;
}
return;
}
/***************************************************************************
* Functions exported by PNETCDF module for coordinating with darshan-core *
***************************************************************************/
static void pnetcdf_begin_shutdown()
{
assert(pnetcdf_runtime);
PNETCDF_LOCK();
/* disable further instrumentation while Darshan shuts down */
instrumentation_disabled = 1;
PNETCDF_UNLOCK();
return;
}
static void pnetcdf_get_output_data(
MPI_Comm mod_comm,
darshan_record_id *shared_recs,
int shared_rec_count,
void **pnetcdf_buf,
int *pnetcdf_buf_sz)
{
struct pnetcdf_file_runtime *file;
int i;
struct darshan_pnetcdf_file *red_send_buf = NULL;
struct darshan_pnetcdf_file *red_recv_buf = NULL;
MPI_Datatype red_type;
MPI_Op red_op;
assert(pnetcdf_runtime);
/* if there are globally shared files, do a shared file reduction */
if(shared_rec_count)
{
/* necessary initialization of shared records */
for(i = 0; i < shared_rec_count; i++)
{
HASH_FIND(hlink, pnetcdf_runtime->file_hash, &shared_recs[i],
sizeof(darshan_record_id), file);
assert(file);
file->file_record->rank = -1;
}
/* sort the array of files descending by rank so that we get all of the
* shared files (marked by rank -1) in a contiguous portion at end
* of the array
*/
qsort(pnetcdf_runtime->file_record_array, pnetcdf_runtime->file_array_ndx,
sizeof(struct darshan_pnetcdf_file), pnetcdf_record_compare);