darshan-mpi-io.c 103 KB
Newer Older
1 2 3 4 5
/*
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

6 7 8
#define _XOPEN_SOURCE 500
#define _GNU_SOURCE /* for tdestroy() */

9
#include "darshan-runtime-config.h"
10

11
#include <stdio.h>
12
#ifdef HAVE_MNTENT_H
13
#include <mntent.h>
14
#endif
15 16 17 18 19 20 21
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <limits.h>
#include <unistd.h>
#include <pthread.h>
#include <sys/types.h>
22
#include <sys/stat.h>
23
#include <sys/vfs.h>
24 25 26 27 28 29
#include <zlib.h>
#include <assert.h>
#include <search.h>

#include "mpi.h"
#include "darshan.h"
30
#include "darshan-dynamic.h"
31
#include "darshan-ext.h"
32

33 34 35 36 37
extern char* __progname;

/* maximum number of memory segments each process will write to the log */
#define CP_MAX_MEM_SEGMENTS 8

38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
/* Some old versions of MPI don't provide all of these COMBINER definitions.  
 * If any are missing then we define them to an arbitrary value just to 
 * prevent compile errors in DATATYPE_INC().
 */
#ifndef MPI_COMBINER_NAMED
    #define MPI_COMBINER_NAMED CP_COMBINER_NAMED
#endif
#ifndef MPI_COMBINER_DUP
    #define MPI_COMBINER_DUP CP_COMBINER_DUP
#endif
#ifndef MPI_COMBINER_CONTIGUOUS
    #define MPI_COMBINER_CONTIGUOUS CP_COMBINER_CONTIGUOUS
#endif
#ifndef MPI_COMBINER_VECTOR
    #define MPI_COMBINER_VECTOR CP_COMBINER_VECTOR
#endif
#ifndef MPI_COMBINER_HVECTOR_INTEGER
    #define MPI_COMBINER_HVECTOR_INTEGER CP_COMBINER_HVECTOR_INTEGER
#endif
#ifndef MPI_COMBINER_HVECTOR
    #define MPI_COMBINER_HVECTOR CP_COMBINER_HVECTOR
#endif
#ifndef MPI_COMBINER_INDEXED
    #define MPI_COMBINER_INDEXED CP_COMBINER_INDEXED
#endif
#ifndef MPI_COMBINER_HINDEXED_INTEGER
    #define MPI_COMBINER_HINDEXED_INTEGER CP_COMBINER_HINDEXED_INTEGER
#endif
#ifndef MPI_COMBINER_HINDEXED
    #define MPI_COMBINER_HINDEXED CP_COMBINER_HINDEXED
#endif
#ifndef MPI_COMBINER_INDEXED_BLOCK
    #define MPI_COMBINER_INDEXED_BLOCK CP_COMBINER_INDEXED_BLOCK
#endif
#ifndef MPI_COMBINER_STRUCT_INTEGER
    #define MPI_COMBINER_STRUCT_INTEGER CP_COMBINER_STRUCT_INTEGER
#endif
#ifndef MPI_COMBINER_STRUCT
    #define MPI_COMBINER_STRUCT CP_COMBINER_STRUCT
#endif
#ifndef MPI_COMBINER_SUBARRAY
    #define MPI_COMBINER_SUBARRAY CP_COMBINER_SUBARRAY
#endif
#ifndef MPI_COMBINER_DARRAY
    #define MPI_COMBINER_DARRAY CP_COMBINER_DARRAY
#endif
#ifndef MPI_COMBINER_F90_REAL
    #define MPI_COMBINER_F90_REAL CP_COMBINER_F90_REAL
#endif
#ifndef MPI_COMBINER_F90_COMPLEX
    #define MPI_COMBINER_F90_COMPLEX CP_COMBINER_F90_COMPLEX
#endif
#ifndef MPI_COMBINER_F90_INTEGER
    #define MPI_COMBINER_F90_INTEGER CP_COMBINER_F90_INTEGER
#endif
#ifndef MPI_COMBINER_RESIZED
    #define MPI_COMBINER_RESIZED CP_COMBINER_RESIZED
#endif

97 98
#define CP_DATATYPE_INC(__file, __datatype) do {\
    int num_integers, num_addresses, num_datatypes, combiner, ret; \
99 100
    ret = DARSHAN_MPI_CALL(PMPI_Type_get_envelope)(__datatype, &num_integers, \
        &num_addresses, &num_datatypes, &combiner); \
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
    if(ret == MPI_SUCCESS) { \
        switch(combiner) { \
            case MPI_COMBINER_NAMED:\
                CP_INC(__file,CP_COMBINER_NAMED,1); break; \
            case MPI_COMBINER_DUP:\
                CP_INC(__file,CP_COMBINER_DUP,1); break; \
            case MPI_COMBINER_CONTIGUOUS:\
                CP_INC(__file,CP_COMBINER_CONTIGUOUS,1); break; \
            case MPI_COMBINER_VECTOR:\
                CP_INC(__file,CP_COMBINER_VECTOR,1); break; \
            case MPI_COMBINER_HVECTOR_INTEGER:\
                CP_INC(__file,CP_COMBINER_HVECTOR_INTEGER,1); break; \
            case MPI_COMBINER_HVECTOR:\
                CP_INC(__file,CP_COMBINER_HVECTOR,1); break; \
            case MPI_COMBINER_INDEXED:\
                CP_INC(__file,CP_COMBINER_INDEXED,1); break; \
            case MPI_COMBINER_HINDEXED_INTEGER:\
                CP_INC(__file,CP_COMBINER_HINDEXED_INTEGER,1); break; \
            case MPI_COMBINER_HINDEXED:\
                CP_INC(__file,CP_COMBINER_HINDEXED,1); break; \
            case MPI_COMBINER_INDEXED_BLOCK:\
                CP_INC(__file,CP_COMBINER_INDEXED_BLOCK,1); break; \
            case MPI_COMBINER_STRUCT_INTEGER:\
                CP_INC(__file,CP_COMBINER_STRUCT_INTEGER,1); break; \
            case MPI_COMBINER_STRUCT:\
                CP_INC(__file,CP_COMBINER_STRUCT,1); break; \
            case MPI_COMBINER_SUBARRAY:\
                CP_INC(__file,CP_COMBINER_SUBARRAY,1); break; \
            case MPI_COMBINER_DARRAY:\
                CP_INC(__file,CP_COMBINER_DARRAY,1); break; \
            case MPI_COMBINER_F90_REAL:\
                CP_INC(__file,CP_COMBINER_F90_REAL,1); break; \
            case MPI_COMBINER_F90_COMPLEX:\
                CP_INC(__file,CP_COMBINER_F90_COMPLEX,1); break; \
            case MPI_COMBINER_F90_INTEGER:\
                CP_INC(__file,CP_COMBINER_F90_INTEGER,1); break; \
            case MPI_COMBINER_RESIZED:\
                CP_INC(__file,CP_COMBINER_RESIZED,1); break; \
        } \
    } \
} while(0)

143 144 145 146 147 148 149
int count_contiguous_blocks_memory(MPI_Datatype datatype, int count);
int count_contiguous_blocks_file(MPI_File fh, MPI_Offset foff1, MPI_Offset foff2);
MPI_Offset func_1_inf(MPI_File fh, MPI_Offset x, int memtype_size);
MPI_Offset func_1(MPI_File fh, MPI_Offset x);

/*
#define CP_RECORD_MPI_WRITE(__ret, __fh, __count, __datatype, __counter, __tm1, __tm2, __voff) do { \
150 151 152
    struct darshan_file_runtime* file; \
    int size = 0; \
    MPI_Aint extent = 0; \
153
    MPI_Offset foff1, foff2; \
154 155 156
    if(__ret != MPI_SUCCESS) break; \
    file = darshan_file_by_fh(__fh); \
    if(!file) break; \
157
    DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size);  \
158
    size = size * __count; \
159
    DARSHAN_MPI_CALL(PMPI_Type_extent)(__datatype, &extent); \
160 161 162 163
    CP_BUCKET_INC(file, CP_SIZE_WRITE_AGG_0_100, size); \
    CP_BUCKET_INC(file, CP_EXTENT_WRITE_0_100, extent); \
    CP_INC(file, __counter, 1); \
    CP_DATATYPE_INC(file, __datatype); \
164
    CP_F_INC_NO_OVERLAP(file, __tm1, __tm2, file->last_mpi_write_end, CP_F_MPI_WRITE_TIME); \
165 166 167
    if(CP_F_VALUE(file, CP_F_WRITE_START_TIMESTAMP) == 0) \
        CP_F_SET(file, CP_F_WRITE_START_TIMESTAMP, __tm1); \
    CP_F_SET(file, CP_F_WRITE_END_TIMESTAMP, __tm2); \
168 169 170 171 172 173 174 175 176
    CP_SET(file, CP_AVG_MEM_DTYPE_SIZE, size); \
    CP_SET(file, CP_AVG_MEM_DTYPE_EXTENT, extent * __count); \
    CP_SET(file, CP_AVG_MEM_DTYPE_BLOCKS, count_contiguous_blocks_memory(__datatype,  __count)); \
    MPI_File_get_byte_offset(__fh, __voff, &foff1); \
    MPI_File_get_byte_offset(__fh, __voff + size, &foff2); \
    CP_SET(file, CP_MIN_FILE_OFFSET, foff1);  \
    CP_SET(file, CP_MAX_FILE_OFFSET, foff2);  \
    CP_SET(file, CP_AVG_FILE_DTYPE_EXTENT, foff2 -foff1);  \
    CP_SET(file, CP_AVG_FILE_DTYPE_BLOCKS, count_contiguous_blocks_file(fh, foff1, foff2 )); \
177
} while(0)
178 179 180 181
*/

static struct darshan_file_runtime* darshan_file_by_fh(MPI_File fh);

fisaila's avatar
fisaila committed
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
void printHints(MPI_Info * mpiHints)
{
    int rank;
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &rank);
    if (rank == 0) {	
    	char key[MPI_MAX_INFO_VAL],
         value[MPI_MAX_INFO_VAL];
    	int  flag, i, nkeys;

    	MPI_Info_get_nkeys(*mpiHints, &nkeys);

    	for (i = 0; i < nkeys; i++) {
        	MPI_Info_get_nthkey(*mpiHints, i, key);
        	MPI_Info_get(*mpiHints, key, MPI_MAX_INFO_VAL-1,
                               value, &flag);
        	fprintf(stdout,"\t%s = %s\n", key, value);
    	}
   }
} /* printHints() */


203 204 205 206 207 208 209
void CP_RECORD_MPI_WRITE(int __ret, MPI_File __fh, int __count, MPI_Datatype __datatype, 
			 int64_t __counter, double __tm1, double __tm2, MPI_Offset __voff) { 
    struct darshan_file_runtime* file; 
    int size = 0; 
    MPI_Aint extent = 0; 
    MPI_Offset foff1, foff2;
    int mem_blocks, file_blocks; //
fisaila's avatar
fisaila committed
210 211 212
    MPI_Info hints;
    MPI_File_get_info(__fh, &hints);
    printHints(&hints);	
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
    if(__ret != MPI_SUCCESS) return; 
    file = darshan_file_by_fh(__fh); 
    if(!file) return; 
    DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size);  
    size = size * __count; 
    DARSHAN_MPI_CALL(PMPI_Type_extent)(__datatype, &extent); 
    CP_BUCKET_INC(file, CP_SIZE_WRITE_AGG_0_100, size); 
    CP_BUCKET_INC(file, CP_EXTENT_WRITE_0_100, extent); 
    CP_INC(file, __counter, 1); 
    CP_DATATYPE_INC(file, __datatype); 
    CP_F_INC_NO_OVERLAP(file, __tm1, __tm2, file->last_mpi_write_end, CP_F_MPI_WRITE_TIME); 
    if(CP_F_VALUE(file, CP_F_WRITE_START_TIMESTAMP) == 0) 
        CP_F_SET(file, CP_F_WRITE_START_TIMESTAMP, __tm1); 
    CP_F_SET(file, CP_F_WRITE_END_TIMESTAMP, __tm2); 
    CP_SET(file, CP_AVG_MEM_DTYPE_SIZE, size); 
    CP_SET(file, CP_AVG_MEM_DTYPE_EXTENT, extent * __count);
    mem_blocks = count_contiguous_blocks_memory(__datatype,  __count); //
    CP_SET(file, CP_AVG_MEM_DTYPE_BLOCKS, mem_blocks); // 
    CP_SET(file, CP_MAX_MEM_DTYPE_SIZE, size); //
    CP_SET(file, CP_MAX_MEM_DTYPE_EXTENT, extent); //
    CP_SET(file, CP_MAX_MEM_DTYPE_BLOCKS, mem_blocks); //
    CP_SET(file, CP_MIN_MEM_DTYPE_SIZE, size); //
    CP_SET(file, CP_MIN_MEM_DTYPE_EXTENT, extent); //
    CP_SET(file, CP_MIN_MEM_DTYPE_BLOCKS, mem_blocks); //
    //MPI_File_get_byte_offset(__fh, __voff, &foff1);
    //MPI_File_get_byte_offset(__fh, __voff + __count, &foff2);
    foff1 = func_1(__fh, __voff); //
    foff2 = func_1_inf(__fh, __voff, size); //
    //    printf("foff1 = %lld foff1_func_1 = %lld foff2 = %lld foff2_func_1_inf = %lld __voff =%lld __count=%d\n", 
    //	   foff1, func_1(__fh, __voff), foff2, func_1_inf(__fh, __voff, size), __voff,  __count);
    file_blocks = count_contiguous_blocks_file(__fh, foff1, foff2 );
    CP_SET(file, CP_MIN_FILE_OFFSET, foff1);  
    CP_SET(file, CP_MAX_FILE_OFFSET, foff2);  
    CP_SET(file, CP_AVG_FILE_DTYPE_EXTENT, foff2 - foff1 + 1);  
    CP_SET(file, CP_AVG_FILE_DTYPE_BLOCKS, file_blocks); 
    CP_SET(file, CP_MAX_FILE_DTYPE_EXTENT, foff2 - foff1 + 1); //
    CP_SET(file, CP_MAX_FILE_DTYPE_BLOCKS, file_blocks);  //
    CP_SET(file, CP_MIN_FILE_DTYPE_EXTENT, foff2 - foff1 + 1); //
    CP_SET(file, CP_MIN_FILE_DTYPE_BLOCKS, file_blocks);  //    
} 

254 255 256 257 258 259 260 261

#define CP_RECORD_MPI_READ(__ret, __fh, __count, __datatype, __counter, __tm1, __tm2) do { \
    struct darshan_file_runtime* file; \
    int size = 0; \
    MPI_Aint extent = 0; \
    if(__ret != MPI_SUCCESS) break; \
    file = darshan_file_by_fh(__fh); \
    if(!file) break; \
262
    DARSHAN_MPI_CALL(PMPI_Type_size)(__datatype, &size);  \
263
    size = size * __count; \
264
    DARSHAN_MPI_CALL(PMPI_Type_extent)(__datatype, &extent); \
265 266 267 268
    CP_BUCKET_INC(file, CP_SIZE_READ_AGG_0_100, size); \
    CP_BUCKET_INC(file, CP_EXTENT_READ_0_100, extent); \
    CP_INC(file, __counter, 1); \
    CP_DATATYPE_INC(file, __datatype); \
269
    CP_F_INC_NO_OVERLAP(file, __tm1, __tm2, file->last_mpi_read_end, CP_F_MPI_READ_TIME); \
270 271 272 273 274
    if(CP_F_VALUE(file, CP_F_READ_START_TIMESTAMP) == 0) \
        CP_F_SET(file, CP_F_READ_START_TIMESTAMP, __tm1); \
    CP_F_SET(file, CP_F_READ_END_TIMESTAMP, __tm2); \
} while(0)

275 276 277
static void cp_log_construct_indices(struct darshan_job_runtime* final_job,
    int rank, int* inout_count, int* lengths, void** pointers, char*
    trailing_data);
278
static int cp_log_write(struct darshan_job_runtime* final_job, int rank, 
279
    char* logfile_name, int count, int* lengths, void** pointers, double start_log_time);
280
static void cp_log_record_hints_and_ver(struct darshan_job_runtime* final_job, int rank);
281
static int cp_log_reduction(struct darshan_job_runtime* final_job, int rank, 
282
    char* logfile_name, MPI_Offset* next_offset);
283 284 285 286 287 288
static void darshan_file_reduce(void* infile_v, 
    void* inoutfile_v, int *len, 
    MPI_Datatype *datatype);
static int cp_log_compress(struct darshan_job_runtime* final_job,
    int rank, int* inout_count, int* lengths, void** pointers);
static int file_compare(const void* a, const void* b);
289 290 291 292 293 294
static int darshan_file_variance(
    struct darshan_file *infile_array,
    struct darshan_file *outfile_array,
    int count, int rank);
static void pairwise_variance_reduce (
    void *invec, void *inoutvec, int *len, MPI_Datatype *dt);
Philip Carns's avatar
Philip Carns committed
295
#if 0
296
static void debug_mounts(const char* mtab_file, const char* out_file);
Philip Carns's avatar
Philip Carns committed
297
#endif
298

299
//static struct darshan_file_runtime* darshan_file_by_fh(MPI_File fh);
300 301
static void darshan_file_close_fh(MPI_File fh);
static struct darshan_file_runtime* darshan_file_by_name_setfh(const char* name, MPI_File fh);
302

303
#define CP_MAX_MNTS 32
Philip Carns's avatar
Philip Carns committed
304 305 306
#define CP_MAX_MNT_PATH 256
#define CP_MAX_MNT_TYPE 32
struct mnt_data
Philip Carns's avatar
Philip Carns committed
307
{
Philip Carns's avatar
Philip Carns committed
308 309 310 311 312 313 314
    int64_t hash;
    int64_t block_size;
    char path[CP_MAX_MNT_PATH];
    char type[CP_MAX_MNT_TYPE];
};
static struct mnt_data mnt_data_array[CP_MAX_MNTS];
static int mnt_data_count = 0;
315

316 317 318 319 320 321 322
struct variance_dt
{
    double n;
    double T;
    double S;
};

323 324
static int epoch_counter = 0;

325 326 327 328 329
// The next two variables used for the context of the file access operation
// e.g. for tracing all the MPI communication from an MPI_File_write
static __thread char *crt_filename = NULL;
static __thread MPI_File *crt_fh = NULL;

330
void darshan_mpi_initialize(int *argc, char ***argv)
331 332 333
{
    int nprocs;
    int rank;
Philip Carns's avatar
Philip Carns committed
334 335
    int timing_flag = 0;
    double init_start, init_time, init_max;
336

337 338
    DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs);
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &rank);
Philip Carns's avatar
Philip Carns committed
339 340 341 342 343 344
    
    if(getenv("DARSHAN_INTERNAL_TIMING"))
        timing_flag = 1;

    if(timing_flag)
        init_start = DARSHAN_MPI_CALL(PMPI_Wtime)();
345 346 347 348 349 350 351 352 353 354

    if(argc && argv)
    {
        darshan_initialize(*argc, *argv, nprocs, rank);
    }
    else
    {
        /* we don't see argc and argv here in fortran */
        darshan_initialize(0, NULL, nprocs, rank);
    }
Philip Carns's avatar
Philip Carns committed
355 356 357 358 359 360 361 362 363 364 365 366
    
    if(timing_flag)
    {
        init_time = DARSHAN_MPI_CALL(PMPI_Wtime)() - init_start;
        DARSHAN_MPI_CALL(PMPI_Reduce)(&init_time, &init_max, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        if(rank == 0)
        {
            printf("#darshan:<op>\t<nprocs>\t<time>\n");
            printf("darshan:init\t%d\t%f\n", nprocs, init_max);
        }
    }
367

368
    return;
369 370
}

371
void darshan_shutdown_epoch(struct darshan_job_runtime* final_job, int timing_flag)
372 373 374
{
    int rank;
    char* logfile_name;
375
    //struct darshan_job_runtime* final_job;
376 377 378 379 380
    double start_log_time = 0;
    int all_ret = 0;
    int local_ret = 0;
    MPI_Offset next_offset = 0;
    char* jobid_str;
381 382
    char* envjobid;
    char* logpath;
383 384 385 386 387 388
    int jobid;
    int index_count = 0;
    int lengths[CP_MAX_MEM_SEGMENTS];
    void* pointers[CP_MAX_MEM_SEGMENTS];
    int ret;
    double red1=0, red2=0, gz1=0, gz2=0, write1=0, write2=0, tm_end=0;
Philip Carns's avatar
Philip Carns committed
389
    double bcst=0;
390
    int nprocs;
391
    time_t start_time_tmp = 0;
392 393
    uint64_t logmod;
    char hname[HOST_NAME_MAX];
394 395 396 397 398
    char* logpath_override = NULL;
#ifdef __CP_LOG_ENV
    char env_check[256];
    char* env_tok;
#endif
399
    uint64_t hlevel;
400
    static int epoch_idx = 0;
401

402 403 404 405 406 407
    //CP_LOCK();
    //if(!darshan_global_job)
    //{
    //    CP_UNLOCK();
    //    return;
    //}
408 409 410
    /* disable further tracing while hanging onto the data so that we can
     * write it out
     */
411
    //final_job = darshan_global_job;
412 413
    //  Moved to the new darshan_shutdown   
    //    darshan_global_job = NULL;
414
    //CP_UNLOCK();
415

416
    start_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462

    /* figure out which access sizes to log */
    darshan_walk_file_accesses(final_job);

    /* if the records have been condensed, then zero out fields that are no
     * longer valid for safety 
     */
    if(final_job->flags & CP_FLAG_CONDENSED && final_job->file_count)
    {
        CP_SET(&final_job->file_runtime_array[0], CP_MODE, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_CONSEC_READS, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_CONSEC_WRITES, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_SEQ_READS, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_SEQ_WRITES, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE1_STRIDE, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE2_STRIDE, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE3_STRIDE, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE4_STRIDE, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE1_COUNT, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE2_COUNT, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE3_COUNT, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_STRIDE4_COUNT, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS1_ACCESS, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS2_ACCESS, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS3_ACCESS, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS4_ACCESS, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS1_COUNT, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS2_COUNT, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS3_COUNT, 0);
        CP_SET(&final_job->file_runtime_array[0], CP_ACCESS4_COUNT, 0);
        
        CP_F_SET(&final_job->file_runtime_array[0], CP_F_OPEN_TIMESTAMP, 0);
        CP_F_SET(&final_job->file_runtime_array[0], CP_F_CLOSE_TIMESTAMP, 0);
        CP_F_SET(&final_job->file_runtime_array[0], CP_F_READ_START_TIMESTAMP, 0);
        CP_F_SET(&final_job->file_runtime_array[0], CP_F_READ_END_TIMESTAMP, 0);
        CP_F_SET(&final_job->file_runtime_array[0], CP_F_WRITE_START_TIMESTAMP, 0);
        CP_F_SET(&final_job->file_runtime_array[0], CP_F_WRITE_END_TIMESTAMP, 0);
    }

    logfile_name = malloc(PATH_MAX);
    if(!logfile_name)
    {
        darshan_finalize(final_job);
        return;
    }

463
    DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, &rank);
464 465 466 467 468 469 470

    /* construct log file name */
    if(rank == 0)
    {
        char cuser[L_cuserid] = {0};
        struct tm* my_tm;

471 472 473 474 475 476 477 478 479 480 481
        /* Use CP_JOBID_OVERRIDE for the env var or CP_JOBID */
        envjobid = getenv(CP_JOBID_OVERRIDE);
        if (!envjobid)
        {
            envjobid = CP_JOBID;
        }

        /* Use CP_LOG_PATH_OVERRIDE for the value or __CP_LOG_PATH */
        logpath = getenv(CP_LOG_PATH_OVERRIDE);
        if (!logpath)
        {
482
#ifdef __CP_LOG_PATH
483
            logpath = __CP_LOG_PATH;
484
#endif
485 486
        }

487
        /* find a job id */
488
        jobid_str = getenv(envjobid);
489 490 491 492 493 494 495 496 497 498 499 500
        if(jobid_str)
        {
            /* in cobalt we can find it in env var */
            ret = sscanf(jobid_str, "%d", &jobid);
        }
        if(!jobid_str || ret != 1)
        {
            /* use pid as fall back */
            jobid = getpid();
        }

        /* break out time into something human readable */
501 502
        start_time_tmp += final_job->log_job.start_time;
        my_tm = localtime(&start_time_tmp);
503

504 505 506 507 508 509 510 511 512 513 514
        /* get the username for this job.  In order we will try each of the
         * following until one of them succeeds:
         *
         * - cuserid()
         * - getenv("LOGNAME")
         * - snprintf(..., geteuid());
         *
         * Note that we do not use getpwuid() because it generally will not
         * work in statically compiled binaries.
         */

515
#ifndef DARSHAN_DISABLE_CUSERID
516
        cuserid(cuser);
517
#endif
518 519 520 521 522 523 524 525 526 527 528 529 530 531

        /* if cuserid() didn't work, then check the environment */
        if (strcmp(cuser, "") == 0)
        {
            char* logname_string;
            logname_string = getenv("LOGNAME");
            if(logname_string)
            {
                strncpy(cuser, logname_string, (L_cuserid-1));
            }

        }

        /* if cuserid() and environment both fail, then fall back to uid */
532 533 534 535 536
        if (strcmp(cuser, "") == 0)
        {
            uid_t uid = geteuid();
            snprintf(cuser, sizeof(cuser), "%u", uid);
        }
537

538
        /* generate a random number to help differentiate the log */
Philip Carns's avatar
Philip Carns committed
539
        hlevel=DARSHAN_MPI_CALL(PMPI_Wtime)() * 1000000;
540
        (void) gethostname(hname, sizeof(hname));
541
        logmod = darshan_hash((void*)hname,strlen(hname),hlevel);
542

543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
        /* see if darshan was configured using the --with-logpath-by-env
         * argument, which allows the user to specify an absolute path to
         * place logs via an env variable.
         */
#ifdef __CP_LOG_ENV
        /* just silently skip if the environment variable list is too big */
        if(strlen(__CP_LOG_ENV) < 256)
        {
            /* copy env variable list to a temporary buffer */
            strcpy(env_check, __CP_LOG_ENV);
            /* tokenize the comma-separated list */
            env_tok = strtok(env_check, ",");
            if(env_tok)
            {
                do
                {
                    /* check each env variable in order */
                    logpath_override = getenv(env_tok); 
                    if(logpath_override)
                    {
                        /* stop as soon as we find a match */
                        break;
                    }
                }while((env_tok = strtok(NULL, ",")));
            }
        }
#endif

571
       
572
        if(logpath_override)
573
        {
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
	    if (epoch_counter > 0)
		ret = snprintf(logfile_name, PATH_MAX, 
		    "%s/%s_%s_id%d_epoch%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
		    logpath_override, 
		    cuser, __progname, jobid, epoch_idx++,
                    (my_tm->tm_mon+1), 
                    my_tm->tm_mday, 
                    (my_tm->tm_hour*60*60 + my_tm->tm_min*60 + my_tm->tm_sec),
                    logmod);
 
	    else
		ret = snprintf(logfile_name, PATH_MAX, 
                    "%s/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
                    logpath_override, 
                    cuser, __progname, jobid,
                    (my_tm->tm_mon+1), 
                    my_tm->tm_mday, 
                    (my_tm->tm_hour*60*60 + my_tm->tm_min*60 + my_tm->tm_sec),
                    logmod);
593 594 595 596 597 598 599 600
            if(ret == (PATH_MAX-1))
            {
                /* file name was too big; squish it down */
                snprintf(logfile_name, PATH_MAX,
                    "%s/id%d.darshan_partial",
                    logpath_override, jobid);
            }
        }
601
        else if(logpath)
602
        {
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
	    if (epoch_counter > 0)	    
		ret = snprintf(logfile_name, PATH_MAX, 
		    "%s/%d/%d/%d/%s_%s_id%d_epoch%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
                    logpath, (my_tm->tm_year+1900), 
                    (my_tm->tm_mon+1), my_tm->tm_mday, 
		    cuser, __progname, jobid, epoch_idx++,
                    (my_tm->tm_mon+1), 
                    my_tm->tm_mday, 
                    (my_tm->tm_hour*60*60 + my_tm->tm_min*60 + my_tm->tm_sec),
                    logmod);
	    else
		ret = snprintf(logfile_name, PATH_MAX, 
		    "%s/%d/%d/%d/%s_%s_id%d_%d-%d-%d-%" PRIu64 ".darshan_partial",
                    logpath, (my_tm->tm_year+1900), 
                    (my_tm->tm_mon+1), my_tm->tm_mday, 
                    cuser, __progname, jobid,
                    (my_tm->tm_mon+1), 
                    my_tm->tm_mday, 
                    (my_tm->tm_hour*60*60 + my_tm->tm_min*60 + my_tm->tm_sec),
                    logmod);

624 625 626 627 628 629 630
            if(ret == (PATH_MAX-1))
            {
                /* file name was too big; squish it down */
                snprintf(logfile_name, PATH_MAX,
                    "%s/id%d.darshan_partial",
                    logpath, jobid);
            }
631
        }
632 633 634 635
        else
        {
            logfile_name[0] = '\0';
        }
636 637 638

        /* add jobid */
        final_job->log_job.jobid = (int64_t)jobid;
639 640 641
    }

    /* broadcast log file name */
Philip Carns's avatar
Philip Carns committed
642
    bcst=DARSHAN_MPI_CALL(PMPI_Wtime)();
643 644
    DARSHAN_MPI_CALL(PMPI_Bcast)(logfile_name, PATH_MAX, MPI_CHAR, 0,
        MPI_COMM_WORLD);
645

646 647 648 649 650 651 652
    if(strlen(logfile_name) == 0)
    {
        /* failed to generate log file name */
        darshan_finalize(final_job);
	return;
    }

653 654 655 656
    final_job->log_job.end_time = time(NULL);

    /* reduce records for shared files */
    if(timing_flag)
657
        red1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
658 659
    local_ret = cp_log_reduction(final_job, rank, logfile_name, 
        &next_offset);
660
    if(timing_flag)
661 662
        red2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
    DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1, MPI_INT, MPI_LOR, 
663 664
        MPI_COMM_WORLD);

665 666 667
    /* if we are using any hints to write the log file, then record those
     * hints in the log file header
     */
668
    cp_log_record_hints_and_ver(final_job, rank);
669

670 671 672 673
    if(all_ret == 0)
    {
        /* collect data to write from local process */
        cp_log_construct_indices(final_job, rank, &index_count, lengths, 
674
            pointers, final_job->trailing_data);
675 676 677 678 679 680
    }

    if(all_ret == 0)
    {
        /* compress data */
        if(timing_flag)
681
            gz1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
682 683 684
        local_ret = cp_log_compress(final_job, rank, &index_count, 
            lengths, pointers);
        if(timing_flag)
685 686 687
            gz2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
        DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1,
            MPI_INT, MPI_LOR, MPI_COMM_WORLD);
688 689 690 691 692 693
    }

    if(all_ret == 0)
    {
        /* actually write out log file */
        if(timing_flag)
694
            write1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
695
        local_ret = cp_log_write(final_job, rank, logfile_name, 
696 697
            index_count, lengths, pointers, start_log_time);
        if(timing_flag)
698 699 700
            write2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
        DARSHAN_MPI_CALL(PMPI_Allreduce)(&local_ret, &all_ret, 1,
            MPI_INT, MPI_LOR, MPI_COMM_WORLD);
701 702
    }

703
    if(rank == 0)
704
    {
705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
        if(all_ret != 0)
        {
            fprintf(stderr, "darshan library warning: unable to write log file %s\n", logfile_name);
            /* if any process failed to write log, then delete the whole 
             * file so we don't leave corrupted results
             */
            unlink(logfile_name);
        }
        else
        {
            /* rename from *.darshan_partial to *-<logwritetime>.darshan.gz,
             * which indicates that this log file is complete and ready for
             * analysis
             */ 
            char* mod_index;
            double end_log_time;
            char* new_logfile_name;

            new_logfile_name = malloc(PATH_MAX);
            if(new_logfile_name)
            {
                new_logfile_name[0] = '\0';
                end_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
                strcat(new_logfile_name, logfile_name);
                mod_index = strstr(new_logfile_name, ".darshan_partial");
                sprintf(mod_index, "_%d.darshan.gz", (int)(end_log_time-start_log_time+1));
                rename(logfile_name, new_logfile_name);
                /* set permissions on log file */
733 734 735
#ifdef __CP_GROUP_READABLE_LOGS
                chmod(new_logfile_name, (S_IRUSR|S_IRGRP)); 
#else
736
                chmod(new_logfile_name, (S_IRUSR)); 
737
#endif
738 739 740
                free(new_logfile_name);
            }
        }
741 742
    }

743 744 745 746
    //  Moved to the new darshan_shutdown   
    // if(final_job->trailing_data)
    //    free(final_job->trailing_data);
    // mnt_data_count = 0;
747
    free(logfile_name);
748 749
    //  Moved to the new darshan_shutdown 
    //    darshan_finalize(final_job);
750 751 752 753 754 755 756
    
    if(timing_flag)
    {
        double red_tm, red_slowest;
        double gz_tm, gz_slowest;
        double write_tm, write_slowest;
        double all_tm, all_slowest;
757
        double bcst_tm, bcst_slowest;
758
        
759
        tm_end = DARSHAN_MPI_CALL(PMPI_Wtime)();
760

Philip Carns's avatar
Philip Carns committed
761
        bcst_tm= red1-bcst;
762 763 764 765 766
        red_tm = red2-red1;
        gz_tm = gz2-gz1;
        write_tm = write2-write1;
        all_tm = tm_end-start_log_time;

767 768 769 770 771 772 773 774 775 776
        DARSHAN_MPI_CALL(PMPI_Reduce)(&red_tm, &red_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&gz_tm, &gz_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&write_tm, &write_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&all_tm, &all_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
        DARSHAN_MPI_CALL(PMPI_Reduce)(&bcst_tm, &bcst_slowest, 1,
            MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
777 778 779

        if(rank == 0)
        {
780
            DARSHAN_MPI_CALL(PMPI_Comm_size)(MPI_COMM_WORLD, &nprocs);
Philip Carns's avatar
Philip Carns committed
781 782 783 784 785
            printf("#darshan:<op>\t<nprocs>\t<time>\n");
            printf("darshan:bcst\t%d\t%f\n", nprocs, bcst_slowest);
            printf("darshan:reduce\t%d\t%f\n", nprocs, red_slowest);
            printf("darshan:gzip\t%d\t%f\n", nprocs, gz_slowest);
            printf("darshan:write\t%d\t%f\n", nprocs, write_slowest);
Philip Carns's avatar
Philip Carns committed
786
            printf("darshan:bcast+reduce+gzip+write\t%d\t%f\n", nprocs, all_slowest);
787 788 789 790 791 792
        }
    }

    return;
}

793 794 795
#ifdef HAVE_MPIIO_CONST
int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh) 
#else
796
int MPI_File_open(MPI_Comm comm, char *filename, int amode, MPI_Info info, MPI_File *fh) 
797
#endif
798 799 800 801 802 803 804 805
{
    int ret;
    struct darshan_file_runtime* file;
    char* tmp;
    int comm_size;
    double tm1, tm2;

    tm1 = darshan_wtime();
806
    ret = DARSHAN_MPI_CALL(PMPI_File_open)(comm, filename, amode, info, fh);
807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
    tm2 = darshan_wtime();

    if(ret == MPI_SUCCESS)
    {
        CP_LOCK();

        /* use ROMIO approach to strip prefix if present */
        /* strip off prefix if there is one, but only skip prefixes
         * if they are greater than length one to allow for windows
         * drive specifications (e.g. c:\...) 
         */
        tmp = strchr(filename, ':');
        if (tmp > filename + 1) {
            filename = tmp + 1;
        }

823 824
        file = darshan_file_by_name_setfh(filename, (*fh));
        if(file)
825 826
        {
            CP_SET(file, CP_MODE, amode);
827
            CP_F_INC_NO_OVERLAP(file, tm1, tm2, file->last_mpi_meta_end, CP_F_MPI_META_TIME);
828
            if(CP_F_VALUE(file, CP_F_OPEN_TIMESTAMP) == 0)
829 830 831
                CP_F_SET(file, CP_F_OPEN_TIMESTAMP,
                DARSHAN_MPI_CALL(PMPI_Wtime)());
            DARSHAN_MPI_CALL(PMPI_Comm_size)(comm, &comm_size);
832 833 834 835 836 837 838 839 840 841 842 843 844
            if(comm_size == 1)
            {
                CP_INC(file, CP_INDEP_OPENS, 1);
            }
            else
            {
                CP_INC(file, CP_COLL_OPENS, 1);
            }
            if(info != MPI_INFO_NULL)
            {
                CP_INC(file, CP_HINTS, 1);
            }
        }
845

846 847 848 849 850 851 852 853 854 855 856 857 858 859
        CP_UNLOCK();
    }

    return(ret);
}

int MPI_File_close(MPI_File *fh) 
{
    struct darshan_file_runtime* file;
    MPI_File tmp_fh = *fh;
    double tm1, tm2;
    int ret;
    
    tm1 = darshan_wtime();
860
    ret = DARSHAN_MPI_CALL(PMPI_File_close)(fh);
861 862 863 864 865 866
    tm2 = darshan_wtime();

    CP_LOCK();
    file = darshan_file_by_fh(tmp_fh);
    if(file)
    {
867
        CP_F_SET(file, CP_F_CLOSE_TIMESTAMP, DARSHAN_MPI_CALL(PMPI_Wtime)());
868
        CP_F_INC_NO_OVERLAP(file, tm1, tm2, file->last_mpi_meta_end, CP_F_MPI_META_TIME);
869
        darshan_file_close_fh(tmp_fh);
870 871 872 873 874 875 876 877 878 879 880 881 882
    }
    CP_UNLOCK();

    return(ret);
}

int MPI_File_sync(MPI_File fh)
{
    int ret;
    struct darshan_file_runtime* file;
    double tm1, tm2;

    tm1 = darshan_wtime();
883
    ret = DARSHAN_MPI_CALL(PMPI_File_sync)(fh);
884 885 886 887 888 889 890
    tm2 = darshan_wtime();
    if(ret == MPI_SUCCESS)
    {
        CP_LOCK();
        file = darshan_file_by_fh(fh);
        if(file)
        {
891
            CP_F_INC_NO_OVERLAP(file, tm1, tm2, file->last_mpi_write_end, CP_F_MPI_WRITE_TIME);
892 893 894 895 896 897 898 899 900
            CP_INC(file, CP_SYNCS, 1);
        }
        CP_UNLOCK();
    }

    return(ret);
}


901 902 903 904
#ifdef HAVE_MPIIO_CONST
int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype, 
    MPI_Datatype filetype, const char *datarep, MPI_Info info)
#else
905 906
int MPI_File_set_view(MPI_File fh, MPI_Offset disp, MPI_Datatype etype, 
    MPI_Datatype filetype, char *datarep, MPI_Info info)
907
#endif
908 909 910 911 912 913
{
    int ret;
    struct darshan_file_runtime* file;
    double tm1, tm2;

    tm1 = darshan_wtime();
914 915
    ret = DARSHAN_MPI_CALL(PMPI_File_set_view)(fh, disp, etype,
        filetype, datarep, info);
916 917 918 919 920 921 922 923 924 925
    tm2 = darshan_wtime();
    if(ret == MPI_SUCCESS)
    {
        CP_LOCK();
        file = darshan_file_by_fh(fh);
        if(file)
        {
            CP_INC(file, CP_VIEWS, 1);
            if(info != MPI_INFO_NULL)
            {
926
                CP_F_INC_NO_OVERLAP(file, tm1, tm2, file->last_mpi_meta_end, CP_F_MPI_META_TIME);
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
                CP_INC(file, CP_HINTS, 1);
            }
            CP_DATATYPE_INC(file, filetype);
        }
        CP_UNLOCK();
    }

    return(ret);
}

int MPI_File_read(MPI_File fh, void *buf, int count, 
    MPI_Datatype datatype, MPI_Status *status)
{
    int ret;
    double tm1, tm2;

943
    crt_fh = &fh;	
944
    tm1 = darshan_wtime();
945
    ret = DARSHAN_MPI_CALL(PMPI_File_read)(fh, buf, count, datatype, status);
946 947 948 949
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_INDEP_READS, tm1, tm2);
    CP_UNLOCK();
950
    crt_fh = NULL;	
951 952 953 954 955 956 957 958 959
    return(ret);
}

int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf,
    int count, MPI_Datatype datatype, MPI_Status *status)
{
    int ret;
    double tm1, tm2;

960
    crt_fh = &fh;
961
    tm1 = darshan_wtime();
962 963
    ret = DARSHAN_MPI_CALL(PMPI_File_read_at)(fh, offset, buf,
        count, datatype, status);
964 965 966 967
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_INDEP_READS, tm1, tm2);
    CP_UNLOCK();
968
    crt_fh = NULL;
969 970 971 972 973 974 975 976 977
    return(ret);
}

int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void * buf,
    int count, MPI_Datatype datatype, MPI_Status * status)
{
    int ret;
    double tm1, tm2;

978
    crt_fh = &fh;
979
    tm1 = darshan_wtime();
980 981
    ret = DARSHAN_MPI_CALL(PMPI_File_read_at_all)(fh, offset, buf,
        count, datatype, status);
982 983 984 985
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_COLL_READS, tm1, tm2);
    CP_UNLOCK();
986
    crt_fh = NULL;
987 988 989 990 991 992 993 994
    return(ret);
}

int MPI_File_read_all(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status *status)
{
    int ret;
    double tm1, tm2;

995
    crt_fh = &fh;
996
    tm1 = darshan_wtime();
997 998
    ret = DARSHAN_MPI_CALL(PMPI_File_read_all)(fh, buf, count,
        datatype, status);
999 1000 1001 1002
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_COLL_READS, tm1, tm2);
    CP_UNLOCK();
1003
    crt_fh = NULL;
1004 1005 1006 1007 1008 1009 1010 1011
    return(ret);
}

int MPI_File_read_shared(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status *status)
{
    int ret;
    double tm1, tm2;

1012
    crt_fh = &fh;
1013
    tm1 = darshan_wtime();
1014 1015
    ret = DARSHAN_MPI_CALL(PMPI_File_read_shared)(fh, buf, count,
        datatype, status);
1016 1017 1018 1019
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_INDEP_READS, tm1, tm2);
    CP_UNLOCK();
1020
    crt_fh = NULL;
1021 1022 1023 1024 1025 1026 1027 1028 1029
    return(ret);
}

int MPI_File_read_ordered(MPI_File fh, void * buf, int count, 
    MPI_Datatype datatype, MPI_Status * status)
{
    int ret;
    double tm1, tm2;

1030
    crt_fh = &fh;
1031
    tm1 = darshan_wtime();
1032 1033
    ret = DARSHAN_MPI_CALL(PMPI_File_read_ordered)(fh, buf, count,
        datatype, status);
1034 1035 1036 1037
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_COLL_READS, tm1, tm2);
    CP_UNLOCK();
1038
    crt_fh = NULL;
1039 1040 1041 1042 1043 1044 1045 1046 1047
    return(ret);
}

int MPI_File_read_at_all_begin(MPI_File fh, MPI_Offset offset, void * buf,
    int count, MPI_Datatype datatype)
{
    int ret;
    double tm1, tm2;

1048
    crt_fh = &fh;
1049
    tm1 = darshan_wtime();
1050 1051
    ret = DARSHAN_MPI_CALL(PMPI_File_read_at_all_begin)(fh, offset, buf,
        count, datatype);
1052 1053 1054 1055
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_SPLIT_READS, tm1, tm2);
    CP_UNLOCK();
1056
    crt_fh = NULL;
1057 1058 1059 1060 1061 1062 1063 1064
    return(ret);
}

int MPI_File_read_all_begin(MPI_File fh, void * buf, int count, MPI_Datatype datatype)
{
    int ret;
    double tm1, tm2;

1065
    crt_fh = &fh;
1066
    tm1 = darshan_wtime();
1067
    ret = DARSHAN_MPI_CALL(PMPI_File_read_all_begin)(fh, buf, count, datatype);
1068 1069 1070 1071
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_SPLIT_READS, tm1, tm2);
    CP_UNLOCK();
1072
    crt_fh = NULL;
1073 1074 1075 1076 1077 1078 1079 1080
    return(ret);
}

int MPI_File_read_ordered_begin(MPI_File fh, void * buf, int count, MPI_Datatype datatype)
{
    int ret;
    double tm1, tm2;

1081
    crt_fh = &fh;
1082
    tm1 = darshan_wtime();
1083 1084
    ret = DARSHAN_MPI_CALL(PMPI_File_read_ordered_begin)(fh, buf, count,
        datatype);
1085 1086 1087 1088
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_SPLIT_READS, tm1, tm2);
    CP_UNLOCK();
1089
    crt_fh = NULL;
1090 1091 1092 1093
    return(ret);
}

int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void * buf,
Philip Carns's avatar
Philip Carns committed
1094
    int count, MPI_Datatype datatype, __D_MPI_REQUEST *request)
1095 1096 1097 1098
{
    int ret;
    double tm1, tm2;

1099
    crt_fh = &fh;
1100
    tm1 = darshan_wtime();
1101 1102
    ret = DARSHAN_MPI_CALL(PMPI_File_iread_at)(fh, offset, buf, count,
        datatype, request);
1103 1104 1105 1106
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_NB_READS, tm1, tm2);
    CP_UNLOCK();
1107
    crt_fh = NULL;
1108 1109 1110
    return(ret);
}

Philip Carns's avatar
Philip Carns committed
1111
int MPI_File_iread(MPI_File fh, void * buf, int count, MPI_Datatype datatype, __D_MPI_REQUEST * request)
1112 1113 1114 1115
{
    int ret;
    double tm1, tm2;

1116
    crt_fh = &fh;
1117
    tm1 = darshan_wtime();
1118
    ret = DARSHAN_MPI_CALL(PMPI_File_iread)(fh, buf, count, datatype, request);
1119 1120 1121 1122
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_NB_READS, tm1, tm2);
    CP_UNLOCK();
1123
    crt_fh = NULL;
1124 1125 1126 1127
    return(ret);
}

int MPI_File_iread_shared(MPI_File fh, void * buf, int count,
Philip Carns's avatar
Philip Carns committed
1128
    MPI_Datatype datatype, __D_MPI_REQUEST * request)
1129 1130 1131 1132
{
    int ret;
    double tm1, tm2;

1133
    crt_fh = &fh;
1134
    tm1 = darshan_wtime();
1135 1136
    ret = DARSHAN_MPI_CALL(PMPI_File_iread_shared)(fh, buf, count,
        datatype, request);
1137 1138 1139 1140
    tm2 = darshan_wtime();
    CP_LOCK();
    CP_RECORD_MPI_READ(ret, fh, count, datatype, CP_NB_READS, tm1, tm2);
    CP_UNLOCK();
1141
    crt_fh = NULL;
1142 1143 1144
    return(ret);
}

1145 1146


1147 1148 1149 1150
#ifdef HAVE_MPIIO_CONST
int MPI_File_write(MPI_File fh, const void *buf, int count, 
    MPI_Datatype datatype, MPI_Status *status)
#else
1151 1152
int MPI_File_write(MPI_File fh, void *buf, int count, 
    MPI_Datatype datatype, MPI_Status *status)
1153
#endif