stub in mpiio module

darshan-runtime/Makefile.in

@@ -51,6 +51,12 @@ lib/darshan-posix.o: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
 lib/darshan-posix.po: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
 	$(CC) $(CFLAGS_SHARED) -c $< -o $@
 
+lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
+	$(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
+	$(CC) $(CFLAGS_SHARED) -c $< -o $@
+
 lib/lookup3.o: lib/lookup3.c
 	$(CC) $(CFLAGS) -c $< -o $@
 
@@ -66,10 +72,10 @@ lib/lookup8.po: lib/lookup8.c
 #%.i: %.c
 #	$(CC) -E $(CFLAGS) -c $< -o $@
 
-lib/libdarshan.a: lib/darshan-posix.o lib/darshan-core.o lib/darshan-common.o lib/lookup3.o lib/lookup8.o
+lib/libdarshan.a: lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-core.o lib/darshan-common.o lib/lookup3.o lib/lookup8.o
 	ar rcs $@ $^
 
-lib/libdarshan.so: lib/darshan-core.po lib/darshan-common.po lib/darshan-posix.po lib/lookup3.po lib/lookup8.po
+lib/libdarshan.so: lib/darshan-core.po lib/darshan-common.po lib/darshan-posix.po lib/darshan-mpiio.po lib/lookup3.po lib/lookup8.po
 	$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ -lpthread -lrt -lz -ldl
 
 install:: all

darshan-runtime/lib/darshan-mpiio.c

+/*
+ *  (C) 2009 by Argonne National Laboratory.
+ *      See COPYRIGHT in top-level directory.
+ */
+
+#include "darshan-runtime-config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <string.h>
+#include <time.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/uio.h>
+#include <sys/mman.h>
+#include <search.h>
+#include <assert.h>
+#include <libgen.h>
+#include <aio.h>
+#define __USE_GNU
+#include <pthread.h>
+
+#include "uthash.h"
+#include "darshan.h"
+#include "darshan-mpiio-log-format.h"
+
+/* TODO: move this stuff to a shared header somewhere */
+#ifdef DARSHAN_PRELOAD
+#define __USE_GNU
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#define DARSHAN_MPI_CALL(func) __real_ ## func
+
+#else
+
+#define DARSHAN_MPI_CALL(func) func
+
+#endif
+
+struct mpiio_runtime_file
+{
+    struct darshan_mpiio_file* file_record;
+    UT_hash_handle hlink;
+};
+
+struct mpiio_runtime_file_ref
+{
+    struct mpiio_runtime_file* file;
+    int fd;
+    UT_hash_handle hlink;
+};
+
+struct mpiio_runtime
+{
+    struct mpiio_runtime_file* file_runtime_array;
+    struct darshan_mpiio_file* file_record_array;
+    int file_array_size;
+    int file_array_ndx;
+    struct mpiio_runtime_file* file_hash;
+    struct mpiio_runtime_file_ref* fd_hash;
+    void *red_buf;
+    int shared_rec_count;
+};
+
+static struct mpiio_runtime *mpiio_runtime = NULL;
+static pthread_mutex_t mpiio_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+static int instrumentation_disabled = 0;
+static int my_rank = -1;
+
+/* TODO: I'm sure these should be applied on all modules */
+/* these are paths that we will not trace */
+static char* exclusions[] = {
+"/etc/",
+"/dev/",
+"/usr/",
+"/bin/",
+"/boot/",
+"/lib/",
+"/opt/",
+"/sbin/",
+"/sys/",
+"/proc/",
+NULL
+};
+
+#define MPIIO_LOCK() pthread_mutex_lock(&mpiio_runtime_mutex)
+#define MPIIO_UNLOCK() pthread_mutex_unlock(&mpiio_runtime_mutex)
+
+#ifdef HAVE_MPIIO_CONST
+int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh) 
+#else
+int MPI_File_open(MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_File *fh) 
+#endif
+{
+    int ret;
+    struct darshan_file_runtime* file;
+    char* tmp;
+    int comm_size;
+    double tm1, tm2;
+
+    tm1 = darshan_core_wtime();
+    ret = DARSHAN_MPI_CALL(PMPI_File_open)(comm, filename, amode, info, fh);
+    tm2 = darshan_core_wtime();
+
+    if(ret == MPI_SUCCESS)
+    {
+        MPIIO_LOCK();
+        /* TODO: initialize fn */
+
+        /* use ROMIO approach to strip prefix if present */
+        /* strip off prefix if there is one, but only skip prefixes
+         * if they are greater than length one to allow for windows
+         * drive specifications (e.g. c:\...) 
+         */
+        tmp = strchr(filename, ':');
+        if (tmp > filename + 1) {
+            filename = tmp + 1;
+        }
+
+        /* TODO: record statistics */
+        printf("HELLO WORLD!\n");
+
+#if 0
+        file = darshan_file_by_name_setfh(filename, (*fh));
+        if(file)
+        {
+            CP_SET(file, CP_MODE, amode);
+            CP_F_INC_NO_OVERLAP(file, tm1, tm2, file->last_mpi_meta_end, CP_F_MPI_META_TIME);
+            if(CP_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0)
+                CP_F_SET(file, CP_F_OPEN_TIMESTAMP,
+                tm1);
+            DARSHAN_MPI_CALL(PMPI_Comm_size)(comm, &comm_size);
+            if(comm_size == 1)
+            {
+                CP_INC(file, CP_INDEP_OPENS, 1);
+            }
+            else
+            {
+                CP_INC(file, CP_COLL_OPENS, 1);
+            }
+            if(info != MPI_INFO_NULL)
+            {
+                CP_INC(file, CP_HINTS, 1);
+            }
+        }
+#endif
+
+        MPIIO_UNLOCK();
+    }
+
+    return(ret);
+}
+
+
+
+#if 0
+static void posix_runtime_initialize(void);
+static struct posix_runtime_file* posix_file_by_name(const char *name);
+static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd);
+static struct posix_runtime_file* posix_file_by_fd(int fd);
+static void posix_file_close_fd(int fd);
+
+static void posix_disable_instrumentation(void);
+static void posix_prepare_for_reduction(darshan_record_id *shared_recs,
+    int *shared_rec_count, void **send_buf, void **recv_buf, int *rec_size);
+static void posix_reduce_records(void* infile_v, void* inoutfile_v,
+    int *len, MPI_Datatype *datatype);
+static void posix_get_output_data(void **buffer, int *size);
+static void posix_shutdown(void);
+
+#define POSIX_SET(__file, __counter, __value) do {\
+    (__file)->file_record->counters[__counter] = __value; \
+} while(0)
+
+#define POSIX_F_SET(__file, __counter, __value) do {\
+    (__file)->file_record->fcounters[__counter] = __value; \
+} while(0)
+
+#define POSIX_INC(__file, __counter, __value) do {\
+    (__file)->file_record->counters[__counter] += __value; \
+} while(0)
+
+#define POSIX_F_INC(__file, __counter, __value) do {\
+    (__file)->file_record->fcounters[__counter] += __value; \
+} while(0)
+
+#define POSIX_F_INC_NO_OVERLAP(__file, __tm1, __tm2, __last, __counter) do { \
+    if(__tm1 > __last) \
+        POSIX_F_INC(__file, __counter, (__tm2-__tm1)); \
+    else \
+        POSIX_F_INC(__file, __counter, (__tm2 - __last)); \
+    if(__tm2 > __last) \
+        __last = __tm2; \
+} while(0)
+
+#define POSIX_VALUE(__file, __counter) \
+    ((__file)->file_record->counters[__counter])
+
+#define POSIX_F_VALUE(__file, __counter) \
+    ((__file)->file_record->fcounters[__counter])
+
+#define POSIX_MAX(__file, __counter, __value) do {\
+    if((__file)->file_record->counters[__counter] < __value) \
+    { \
+        (__file)->file_record->counters[__counter] = __value; \
+    } \
+} while(0)
+
+#define POSIX_RECORD_OPEN(__ret, __path, __mode, __stream_flag, __tm1, __tm2) do { \
+    struct posix_runtime_file* file; \
+    char* exclude; \
+    int tmp_index = 0; \
+    if(__ret < 0) break; \
+    while((exclude = exclusions[tmp_index])) { \
+        if(!(strncmp(exclude, __path, strlen(exclude)))) \
+            break; \
+        tmp_index++; \
+    } \
+    if(exclude) break; \
+    file = posix_file_by_name_setfd(__path, __ret); \
+    if(!file) break; \
+    file->file_record->rank = my_rank; \
+    if(__mode) \
+        POSIX_SET(file, CP_MODE, __mode); \
+    if(__stream_flag)\
+        POSIX_INC(file, CP_POSIX_FOPENS, 1); \
+    else \
+        POSIX_INC(file, CP_POSIX_OPENS, 1); \
+    if(POSIX_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0) \
+        POSIX_F_SET(file, CP_F_OPEN_TIMESTAMP, __tm1); \
+} while(0)
+
+int DARSHAN_DECL(open)(const char *path, int flags, ...)
+{
+    int mode = 0;
+    int ret;
+    double tm1, tm2;
+
+    MAP_OR_FAIL(open);
+
+    if(flags & O_CREAT) 
+    {
+        va_list arg;
+        va_start(arg, flags);
+        mode = va_arg(arg, int);
+        va_end(arg);
+
+        tm1 = darshan_core_wtime();
+        ret = __real_open(path, flags, mode);
+        tm2 = darshan_core_wtime();
+    }
+    else
+    {
+        tm1 = darshan_core_wtime();
+        ret = __real_open(path, flags);
+        tm2 = darshan_core_wtime();
+    }
+
+    POSIX_LOCK();
+    posix_runtime_initialize();
+
+    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
+
+    POSIX_UNLOCK();
+
+    return(ret);
+}
+
+int DARSHAN_DECL(open64)(const char *path, int flags, ...)
+{
+    int mode = 0;
+    int ret;
+    double tm1, tm2;
+
+    MAP_OR_FAIL(open64);
+
+    if(flags & O_CREAT)
+    {
+        va_list arg;
+        va_start(arg, flags);
+        mode = va_arg(arg, int);
+        va_end(arg);
+
+        tm1 = darshan_core_wtime();
+        ret = __real_open64(path, flags, mode);
+        tm2 = darshan_core_wtime();
+    }
+    else
+    {
+        tm1 = darshan_core_wtime();
+        ret = __real_open64(path, flags);
+        tm2 = darshan_core_wtime();
+    }
+
+    POSIX_LOCK();
+    posix_runtime_initialize();
+
+    POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
+    POSIX_UNLOCK();
+
+    return(ret);
+}
+
+int DARSHAN_DECL(close)(int fd)
+{
+    struct posix_runtime_file* file;
+    double tm1, tm2;
+    int ret;
+
+    MAP_OR_FAIL(close);
+
+    tm1 = darshan_core_wtime();
+    ret = __real_close(fd);
+    tm2 = darshan_core_wtime();
+
+    POSIX_LOCK();
+    posix_runtime_initialize();
+
+    file = posix_file_by_fd(fd);
+    if(file)
+    {
+        POSIX_F_SET(file, CP_F_CLOSE_TIMESTAMP, darshan_core_wtime());
+        posix_file_close_fd(fd);
+    }
+    POSIX_UNLOCK();    
+
+    return(ret);
+}
+
+/* ***************************************************** */
+
+static void posix_runtime_initialize()
+{
+    char *alignstr;
+    int tmpval;
+    int ret;
+    int mem_limit;
+    struct darshan_module_funcs posix_mod_fns =
+    {
+        .disable_instrumentation = &posix_disable_instrumentation,
+        .prepare_for_reduction = &posix_prepare_for_reduction,
+        .reduce_records = &posix_reduce_records,
+        .get_output_data = &posix_get_output_data,
+        .shutdown = &posix_shutdown
+    };
+
+    /* don't do anything if already initialized or instrumenation is disabled */
+    if(posix_runtime || instrumentation_disabled)
+        return;
+
+    /* register the posix module with darshan core */
+    darshan_core_register_module(
+        DARSHAN_POSIX_MOD,
+        &posix_mod_fns,
+        &mem_limit);
+
+    /* return if no memory assigned by darshan core */
+    if(mem_limit == 0)
+        return;
+
+    posix_runtime = malloc(sizeof(*posix_runtime));
+    if(!posix_runtime)
+        return;
+    memset(posix_runtime, 0, sizeof(*posix_runtime));
+
+    /* set maximum number of file records according to max memory limit */
+    /* NOTE: maximum number of records is based on the size of a posix file record */
+    posix_runtime->file_array_size = mem_limit / sizeof(struct darshan_posix_file);
+    posix_runtime->file_array_ndx = 0;
+
+    /* allocate array of runtime file records */
+    posix_runtime->file_runtime_array = malloc(posix_runtime->file_array_size *
+                                               sizeof(struct posix_runtime_file));
+    posix_runtime->file_record_array = malloc(posix_runtime->file_array_size *
+                                              sizeof(struct darshan_posix_file));
+    if(!posix_runtime->file_runtime_array || !posix_runtime->file_record_array)
+    {
+        posix_runtime->file_array_size = 0;
+        return;
+    }
+    memset(posix_runtime->file_runtime_array, 0, posix_runtime->file_array_size *
+           sizeof(struct posix_runtime_file));
+    memset(posix_runtime->file_record_array, 0, posix_runtime->file_array_size *
+           sizeof(struct darshan_posix_file));
+
+    /* TODO: can we move this out of here? perhaps register_module returns rank? */
+    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &my_rank);
+
+    return;
+}
+
+static struct posix_runtime_file* posix_file_by_name(const char *name)
+{
+    struct posix_runtime_file *file = NULL;
+    char *newname = NULL;
+    darshan_record_id file_id;
+
+    if(!posix_runtime || instrumentation_disabled)
+        return(NULL);
+
+    newname = darshan_clean_file_path(name);
+    if(!newname)
+        newname = (char*)name;
+
+    /* get a unique id for this file from darshan core */
+    darshan_core_register_record(
+        (void*)newname,
+        strlen(newname),
+        1,
+        DARSHAN_POSIX_MOD,
+        &file_id);
+
+    /* search the hash table for this file record, and return if found */
+    HASH_FIND(hlink, posix_runtime->file_hash, &file_id, sizeof(darshan_record_id), file);
+    if(file)
+    {
+        if(newname != name)
+            free(newname);
+        return(file);
+    }
+
+    if(posix_runtime->file_array_ndx < posix_runtime->file_array_size);
+    {
+        /* no existing record, assign a new file record from the global array */
+        file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
+        file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
+        file->file_record->f_id = file_id;
+
+        /* add new record to file hash table */
+        HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
+
+        posix_runtime->file_array_ndx++;
+    }
+
+    if(newname != name)
+        free(newname);
+    return(file);
+}
+
+static struct posix_runtime_file* posix_file_by_name_setfd(const char* name, int fd)
+{
+    struct posix_runtime_file* file;
+    struct posix_runtime_file_ref* ref;
+
+    if(!posix_runtime || instrumentation_disabled)
+        return(NULL);
+
+    /* find file record by name first */
+    file = posix_file_by_name(name);
+
+    if(!file)
+        return(NULL);
+
+    /* search hash table for existing file ref for this fd */
+    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
+    if(ref)
+    {
+        /* we have a reference.  Make sure it points to the correct file
+         * and return it
+         */
+        ref->file = file;
+        return(file);
+    }
+
+    /* if we hit this point, then we don't have a reference for this fd
+     * in the table yet.  Add it.
+     */
+    ref = malloc(sizeof(*ref));
+    if(!ref)
+        return(NULL);
+    memset(ref, 0, sizeof(*ref));
+
+    ref->file = file;
+    ref->fd = fd;    
+    HASH_ADD(hlink, posix_runtime->fd_hash, fd, sizeof(int), ref);
+
+    return(file);
+}
+
+static struct posix_runtime_file* posix_file_by_fd(int fd)
+{
+    struct posix_runtime_file_ref* ref;
+
+    if(!posix_runtime || instrumentation_disabled)
+        return(NULL);
+
+    /* search hash table for existing file ref for this fd */
+    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
+    if(ref)
+        return(ref->file);
+
+    return(NULL);
+}
+
+static void posix_file_close_fd(int fd)
+{
+    struct posix_runtime_file_ref* ref;
+
+    if(!posix_runtime || instrumentation_disabled)
+        return;
+
+    /* search hash table for this fd */
+    HASH_FIND(hlink, posix_runtime->fd_hash, &fd, sizeof(int), ref);
+    if(ref)
+    {
+        /* we have a reference, delete it */
+        HASH_DELETE(hlink, posix_runtime->fd_hash, ref);
+        free(ref);
+    }
+
+    return;
+}
+
+static int posix_file_compare(const void* a, const void* b)
+{
+    const struct darshan_posix_file* f_a = a;
+    const struct darshan_posix_file* f_b = b;
+
+    if(f_a->rank < f_b->rank)
+        return 1;
+    if(f_a->rank > f_b->rank)
+        return -1;
+
+    return 0;
+}
+
+/* ***************************************************** */
+
+static void posix_disable_instrumentation()
+{
+    assert(posix_runtime);
+
+    POSIX_LOCK();
+    instrumentation_disabled = 1;
+    POSIX_UNLOCK();
+
+    return;
+}
+
+static void posix_prepare_for_reduction(
+    darshan_record_id *shared_recs,
+    int *shared_rec_count,
+    void **send_buf,
+    void **recv_buf,
+    int *rec_size)
+{
+    struct posix_runtime_file *file;
+    struct darshan_posix_file *tmp_array;
+    int i;
+
+    assert(posix_runtime);
+
+    /* necessary initialization of shared records (e.g., change rank to -1) */
+    for(i = 0; i < *shared_rec_count; i++)
+    {
+        HASH_FIND(hlink, posix_runtime->file_hash, &shared_recs[i],
+            sizeof(darshan_record_id), file);
+        assert(file);
+
+        /* TODO: any initialization before reduction */
+        file->file_record->rank = -1;
+    }
+
+    /* sort the array of files descending by rank so that we get all of the 
+     * shared files (marked by rank -1) in a contiguous portion at end 
+     * of the array
+     */
+    qsort(posix_runtime->file_record_array, posix_runtime->file_array_ndx,
+        sizeof(struct darshan_posix_file), posix_file_compare);
+
+    /* make *send_buf point to the shared files at the end of sorted array */
+    *send_buf =
+        &(posix_runtime->file_record_array[posix_runtime->file_array_ndx-(*shared_rec_count)]);
+
+    /* allocate memory for the reduction output on rank 0 */
+    if(my_rank == 0)
+    {
+        *recv_buf = malloc(*shared_rec_count * sizeof(struct darshan_posix_file));
+        if(!(*recv_buf))
+            return;
+    }
+
+    *rec_size = sizeof(struct darshan_posix_file);
+
+    /* TODO: cleaner way to do this? */
+    if(my_rank == 0)
+        posix_runtime->red_buf = *recv_buf;
+    posix_runtime->shared_rec_count = *shared_rec_count;
+
+    return;
+}
+
+static void posix_reduce_records(
+    void* infile_v,
+    void* inoutfile_v,
+    int *len,
+    MPI_Datatype *datatype)
+{
+    struct darshan_posix_file tmp_file;
+    struct darshan_posix_file *infile = infile_v;
+    struct darshan_posix_file *inoutfile = inoutfile_v;
+    int i;
+
+    assert(posix_runtime);
+
+    for(i = 0; i < *len; i++)
+    {
+        memset(&tmp_file, 0, sizeof(struct darshan_posix_file));
+
+        tmp_file.f_id = infile->f_id;
+        tmp_file.rank = -1;
+