Skip to content

GitLab

Commit c3c5e597 authored by Shane Snyder's avatar Shane Snyder
Browse files

more mmap logic and instrumentation for testing

parent 0d84b5e5
Hide whitespace changes
Inline Side-by-side
Showing with 230 additions and 434 deletions
darshan-runtime/Makefile.in
View file @ c3c5e597
......@@ -33,15 +33,8 @@ CFLAGS_SHARED = -DDARSHAN_CONFIG_H=\"darshan-runtime-config.h\" -I . -I$(srcdir)
LIBS = -lz @LIBBZ2@
static-mod-objs = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o
dynamic-mod-objs = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po
ifdef DARSHAN_USE_BGQ
static-mod-objs += lib/darshan-bgq.o
dynamic-mod-objs += lib/darshan-bgq.po
CFLAGS += -DDARSHAN_BGQ
CFLAGS_SHARED += -DDARSHAN_BGQ
endif
static-mod-objs = lib/darshan-posix.o
dynamic-mod-objs = lib/darshan-posix.po
lib::
@mkdir -p $@
......
darshan-runtime/lib/darshan-core.c
View file @ c3c5e597
......@@ -122,7 +122,7 @@ void darshan_core_initialize(int argc, char **argv)
struct darshan_core_runtime *init_core = NULL;
int internal_timing_flag = 0;
double init_start, init_time, init_max;
char *mmap_log_name = "darshan-log.out";
char mmap_log_name[PATH_MAX];
int mmap_fd;
int mmap_size;
int sys_page_size;
......@@ -176,6 +176,26 @@ void darshan_core_initialize(int argc, char **argv)
memset(init_core, 0, sizeof(*init_core));
init_core->wtime_offset = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* Use DARSHAN_JOBID_OVERRIDE for the env var for __DARSHAN_JOBID */
envstr = getenv(DARSHAN_JOBID_OVERRIDE);
if(!envstr)
{
envstr = __DARSHAN_JOBID;
}
/* find a job id */
jobid_str = getenv(envstr);
if(jobid_str)
{
/* in cobalt we can find it in env var */
ret = sscanf(jobid_str, "%d", &jobid);
}
if(!jobid_str || ret != 1)
{
/* use pid as fall back */
jobid = getpid();
}
sys_page_size = sysconf(_SC_PAGESIZE);
assert(sys_page_size > 0);
......@@ -187,7 +207,13 @@ void darshan_core_initialize(int argc, char **argv)
if(mmap_size % sys_page_size)
mmap_size = ((mmap_size / sys_page_size) + 1) * sys_page_size;
/* TODO: logfile name should have process rank in it for uniqueness */
/* construct a unique temporary log file name for this process
* to write mmap log data to
*/
snprintf(mmap_log_name, PATH_MAX, "/tmp/darshan_job%d.%d",
jobid, my_rank);
/* create the temporary mmapped darshan log */
mmap_fd = open(mmap_log_name, O_CREAT|O_RDWR|O_EXCL , 0644);
if(mmap_fd < 0)
{
......@@ -238,26 +264,6 @@ void darshan_core_initialize(int argc, char **argv)
init_core->mmap_job_p->uid = getuid();
init_core->mmap_job_p->start_time = time(NULL);
init_core->mmap_job_p->nprocs = nprocs;
/* Use DARSHAN_JOBID_OVERRIDE for the env var for __DARSHAN_JOBID */
envstr = getenv(DARSHAN_JOBID_OVERRIDE);
if(!envstr)
{
envstr = __DARSHAN_JOBID;
}
/* find a job id */
jobid_str = getenv(envstr);
if(jobid_str)
{
/* in cobalt we can find it in env var */
ret = sscanf(jobid_str, "%d", &jobid);
}
if(!jobid_str || ret != 1)
{
/* use pid as fall back */
jobid = getpid();
}
init_core->mmap_job_p->jobid = (int64_t)jobid;
/* if we are using any hints to write the log file, then record those
......@@ -299,42 +305,17 @@ void darshan_core_initialize(int argc, char **argv)
void darshan_core_shutdown()
{
return;
#if 0
int i;
char *logfile_name;
struct darshan_core_runtime *final_core;
int internal_timing_flag = 0;
char *envjobid;
char *jobid_str;
int jobid;
struct tm *start_tm;
time_t start_time_tmp;
int ret = 0;
int all_ret = 0;
int64_t first_start_time;
int64_t last_end_time;
int local_mod_use[DARSHAN_MAX_MODS] = {0};
int global_mod_use_count[DARSHAN_MAX_MODS] = {0};
darshan_record_id shared_recs[DARSHAN_CORE_MAX_RECORDS] = {0};
double start_log_time;
double open1, open2;
double job1, job2;
double rec1, rec2;
double mod1[DARSHAN_MAX_MODS] = {0};
double mod2[DARSHAN_MAX_MODS] = {0};
double header1, header2;
double tm_end;
uint64_t gz_fp = 0;
unsigned char tmp_partial_flag;
MPI_File log_fh;
MPI_Status status;
if(getenv("DARSHAN_INTERNAL_TIMING"))
internal_timing_flag = 1;
start_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
if(internal_timing_flag)
start_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* disable darhan-core while we shutdown */
DARSHAN_CORE_LOCK();
......@@ -346,413 +327,38 @@ void darshan_core_shutdown()
final_core = darshan_core;
darshan_core = NULL;
/* we also need to set which modules were registered on this process and
* call into those modules and give them a chance to perform any necessary
* pre-shutdown steps.
*/
for(i = 0; i < DARSHAN_MAX_MODS; i++)
{
if(final_core->mod_array[i])
{
local_mod_use[i] = 1;
final_core->mod_array[i]->mod_funcs.begin_shutdown();
}
}
DARSHAN_CORE_UNLOCK();
logfile_name = malloc(PATH_MAX);
if(!logfile_name)
{
darshan_core_cleanup(final_core);
return;
}
/* set darshan job id/metadata and constuct log file name on rank 0 */
if(my_rank == 0)
{
/* Use DARSHAN_JOBID_OVERRIDE for the env var for __DARSHAN_JOBID */
envjobid = getenv(DARSHAN_JOBID_OVERRIDE);
if(!envjobid)
{
envjobid = __DARSHAN_JOBID;
}
/* find a job id */
jobid_str = getenv(envjobid);
if(jobid_str)
{
/* in cobalt we can find it in env var */
ret = sscanf(jobid_str, "%d", &jobid);
}
if(!jobid_str || ret != 1)
{
/* use pid as fall back */
jobid = getpid();
}
final_core->log_job.jobid = (int64_t)jobid;
/* if we are using any hints to write the log file, then record those
* hints with the darshan job information
*/
darshan_log_record_hints_and_ver(final_core);
/* use human readable start time format in log filename */
start_time_tmp = final_core->log_job.start_time;
start_tm = localtime(&start_time_tmp);
/* construct log file name */
darshan_get_logfile_name(logfile_name, jobid, start_tm);
}
/* broadcast log file name */
DARSHAN_MPI_CALL(PMPI_Bcast)(logfile_name, PATH_MAX, MPI_CHAR, 0,
MPI_COMM_WORLD);
if(strlen(logfile_name) == 0)
{
/* failed to generate log file name */
free(logfile_name);
darshan_core_cleanup(final_core);
return;
}
final_core->log_job.end_time = time(NULL);
/* reduce to report first start time and last end time across all ranks
* at rank 0
*/
DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job.start_time, &first_start_time, 1, MPI_LONG_LONG, MPI_MIN, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job.end_time, &last_end_time, 1, MPI_LONG_LONG, MPI_MAX, 0, MPI_COMM_WORLD);
if(my_rank == 0)
{
final_core->log_job.start_time = first_start_time;
final_core->log_job.end_time = last_end_time;
}
/* reduce the number of times a module was opened globally and bcast to everyone */
DARSHAN_MPI_CALL(PMPI_Allreduce)(local_mod_use, global_mod_use_count, DARSHAN_MAX_MODS, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
/* get a list of records which are shared across all processes */
darshan_get_shared_records(final_core, shared_recs);
if(internal_timing_flag)
open1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* collectively open the darshan log file */
ret = darshan_log_open_all(logfile_name, &log_fh);
if(internal_timing_flag)
open2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* error out if unable to open log file */
DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
MPI_LOR, MPI_COMM_WORLD);
if(all_ret != 0)
{
if(my_rank == 0)
{
fprintf(stderr, "darshan library warning: unable to open log file %s\n",
logfile_name);
unlink(logfile_name);
}
free(logfile_name);
darshan_core_cleanup(final_core);
return;
}
if(internal_timing_flag)
job1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* rank 0 is responsible for writing the compressed darshan job information */
if(my_rank == 0)
{
void *pointers[2] = {&final_core->log_job, final_core->trailing_data};
int lengths[2] = {sizeof(struct darshan_job), strlen(final_core->trailing_data)};
int comp_buf_sz = 0;
/* compress the job info and the trailing mount/exe data */
all_ret = darshan_deflate_buffer(pointers, lengths, 2,
final_core->comp_buf, &comp_buf_sz);
if(all_ret)
{
fprintf(stderr, "darshan library warning: unable to compress job data\n");
unlink(logfile_name);
}
else
{
/* write the job information, preallocing space for the log header */
gz_fp += sizeof(struct darshan_header);
all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, gz_fp,
final_core->comp_buf, comp_buf_sz, MPI_BYTE, &status);
if(all_ret != MPI_SUCCESS)
{
fprintf(stderr, "darshan library warning: unable to write job data to log file %s\n",
logfile_name);
unlink(logfile_name);
}
gz_fp += comp_buf_sz;
}
}
/* error out if unable to write job information */
DARSHAN_MPI_CALL(PMPI_Bcast)(&all_ret, 1, MPI_INT, 0, MPI_COMM_WORLD);
if(all_ret != 0)
{
free(logfile_name);
darshan_core_cleanup(final_core);
return;
}
if(internal_timing_flag)
job2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
if(internal_timing_flag)
rec1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* write the record name->id hash to the log file */
final_core->log_header.rec_map.off = gz_fp;
ret = darshan_log_write_record_hash(log_fh, final_core, &gz_fp);
final_core->log_header.rec_map.len = gz_fp - final_core->log_header.rec_map.off;
/* error out if unable to write record hash */
DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
MPI_LOR, MPI_COMM_WORLD);
if(all_ret != 0)
{
if(my_rank == 0)
{
fprintf(stderr, "darshan library warning: unable to write record hash to log file %s\n",
logfile_name);
unlink(logfile_name);
}
free(logfile_name);
darshan_core_cleanup(final_core);
return;
}
if(internal_timing_flag)
rec2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* loop over globally used darshan modules and:
* - perform shared file reductions, if possible
* - get final output buffer
* - compress (zlib) provided output buffer
* - append compressed buffer to log file
* - add module index info (file offset/length) to log header
* - shutdown the module
*/
for(i = 0; i < DARSHAN_MAX_MODS; i++)
{
struct darshan_core_module* this_mod = final_core->mod_array[i];
struct darshan_core_record_ref *ref = NULL;
darshan_record_id mod_shared_recs[DARSHAN_CORE_MAX_RECORDS];
int mod_shared_rec_cnt = 0;
void* mod_buf = NULL;
int mod_buf_sz = 0;
int j;
if(global_mod_use_count[i] == 0)
{
if(my_rank == 0)
{
final_core->log_header.mod_map[i].off = 0;
final_core->log_header.mod_map[i].len = 0;
}
continue;
}
if(internal_timing_flag)
mod1[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* set the shared file list for this module */
memset(mod_shared_recs, 0, DARSHAN_CORE_MAX_RECORDS * sizeof(darshan_record_id));
for(j = 0; j < DARSHAN_CORE_MAX_RECORDS && shared_recs[j] != 0; j++)
{
HASH_FIND(hlink, final_core->rec_hash, &shared_recs[j],
sizeof(darshan_record_id), ref);
assert(ref);
if(DARSHAN_CORE_MOD_ISSET(ref->global_mod_flags, i))
{
mod_shared_recs[mod_shared_rec_cnt++] = shared_recs[j];
}
}
/* if module is registered locally, get the corresponding output buffer
*
* NOTE: this function can be used to run collective operations across
* modules, if there are file records shared globally.
*/
if(this_mod)
{
this_mod->mod_funcs.get_output_data(MPI_COMM_WORLD, mod_shared_recs,
mod_shared_rec_cnt, &mod_buf, &mod_buf_sz);
}
/* append this module's data to the darshan log */
final_core->log_header.mod_map[i].off = gz_fp;
ret = darshan_log_append_all(log_fh, final_core, mod_buf, mod_buf_sz, &gz_fp);
final_core->log_header.mod_map[i].len =
gz_fp - final_core->log_header.mod_map[i].off;
/* error out if the log append failed */
DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
MPI_LOR, MPI_COMM_WORLD);
if(all_ret != 0)
{
if(my_rank == 0)
{
fprintf(stderr,
"darshan library warning: unable to write %s module data to log file %s\n",
darshan_module_names[i], logfile_name);
unlink(logfile_name);
}
free(logfile_name);
darshan_core_cleanup(final_core);
return;
}
/* shutdown module if registered locally */
if(this_mod)
{
this_mod->mod_funcs.shutdown();
}
if(internal_timing_flag)
mod2[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
}
/* run a reduction to determine if any application processes had to set the
* partial flag. this happens when a process has tracked too many records
* at once and cannot track new records
*/
DARSHAN_MPI_CALL(PMPI_Reduce)(&(final_core->log_header.partial_flag),
&tmp_partial_flag, 1, MPI_UNSIGNED_CHAR, MPI_MAX, 0, MPI_COMM_WORLD);
if(internal_timing_flag)
header1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* rank 0 is responsible for writing the log header */
if(my_rank == 0)
{
/* initialize the remaining header fields */
strcpy(final_core->log_header.version_string, DARSHAN_LOG_VERSION);
final_core->log_header.magic_nr = DARSHAN_MAGIC_NR;
final_core->log_header.comp_type = DARSHAN_ZLIB_COMP;
final_core->log_header.partial_flag = tmp_partial_flag;
all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, 0, &(final_core->log_header),
sizeof(struct darshan_header), MPI_BYTE, &status);
if(all_ret != MPI_SUCCESS)
{
fprintf(stderr, "darshan library warning: unable to write header to log file %s\n",
logfile_name);
unlink(logfile_name);
}
}
/* error out if unable to write log header */
DARSHAN_MPI_CALL(PMPI_Bcast)(&all_ret, 1, MPI_INT, 0, MPI_COMM_WORLD);
if(all_ret != 0)
{
free(logfile_name);
darshan_core_cleanup(final_core);
return;
}
if(internal_timing_flag)
header2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
final_core->mmap_job_p->end_time = time(NULL);
DARSHAN_MPI_CALL(PMPI_File_close)(&log_fh);
/* if we got this far, there are no errors, so rename from *.darshan_partial
* to *-<logwritetime>.darshan, which indicates that this log file is
* complete and ready for analysis
*/
if(my_rank == 0)
{
if(getenv("DARSHAN_LOGFILE"))
{
#ifdef __DARSHAN_GROUP_READABLE_LOGS
chmod(logfile_name, (S_IRUSR|S_IRGRP));
#else
chmod(logfile_name, (S_IRUSR));
#endif
}
else
{
char* tmp_index;
double end_log_time;
char* new_logfile_name;
new_logfile_name = malloc(PATH_MAX);
if(new_logfile_name)
{
new_logfile_name[0] = '\0';
end_log_time = DARSHAN_MPI_CALL(PMPI_Wtime)();
strcat(new_logfile_name, logfile_name);
tmp_index = strstr(new_logfile_name, ".darshan_partial");
sprintf(tmp_index, "_%d.darshan", (int)(end_log_time-start_log_time+1));
rename(logfile_name, new_logfile_name);
/* set permissions on log file */
#ifdef __DARSHAN_GROUP_READABLE_LOGS
chmod(new_logfile_name, (S_IRUSR|S_IRGRP));
#else
chmod(new_logfile_name, (S_IRUSR));
#endif
free(new_logfile_name);
}
}
}
free(logfile_name);
darshan_core_cleanup(final_core);
if(internal_timing_flag)
{
double open_tm, open_slowest;
double header_tm, header_slowest;
double job_tm, job_slowest;
double rec_tm, rec_slowest;
double mod_tm[DARSHAN_MAX_MODS], mod_slowest[DARSHAN_MAX_MODS];
double all_tm, all_slowest;
tm_end = DARSHAN_MPI_CALL(PMPI_Wtime)();
open_tm = open2 - open1;
header_tm = header2 - header1;
job_tm = job2 - job1;
rec_tm = rec2 - rec1;
all_tm = tm_end - start_log_time;
for(i = 0;i < DARSHAN_MAX_MODS; i++)
{
mod_tm[i] = mod2[i] - mod1[i];
}
DARSHAN_MPI_CALL(PMPI_Reduce)(&open_tm, &open_slowest, 1,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Reduce)(&header_tm, &header_slowest, 1,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Reduce)(&job_tm, &job_slowest, 1,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Reduce)(&rec_tm, &rec_slowest, 1,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Reduce)(&all_tm, &all_slowest, 1,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
DARSHAN_MPI_CALL(PMPI_Reduce)(mod_tm, mod_slowest, DARSHAN_MAX_MODS,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if(my_rank == 0)
{
fprintf(stderr, "#darshan:<op>\t<nprocs>\t<time>\n");
fprintf(stderr, "darshan:log_open\t%d\t%f\n", nprocs, open_slowest);
fprintf(stderr, "darshan:job_write\t%d\t%f\n", nprocs, job_slowest);
fprintf(stderr, "darshan:hash_write\t%d\t%f\n", nprocs, rec_slowest);
fprintf(stderr, "darshan:header_write\t%d\t%f\n", nprocs, header_slowest);
for(i = 0; i < DARSHAN_MAX_MODS; i++)
{
if(global_mod_use_count[i])
fprintf(stderr, "darshan:%s_shutdown\t%d\t%f\n", darshan_module_names[i],
nprocs, mod_slowest[i]);
}
fprintf(stderr, "darshan:core_shutdown\t%d\t%f\n", nprocs, all_slowest);
}
}
return;
#endif
}
/* *********************************** */
......
darshan-runtime/lib/darshan-posix.c
View file @ c3c5e597
......@@ -346,6 +346,16 @@ static void posix_shutdown(void);
(__file)->file_record->counters[POSIX_STATS] += 1; \
} while(0)
static double psx_wrap_time = 0.0;
#define PSX_PRE_WRAP() \
double pre_wrap = darshan_core_wtime();
#define PSX_POST_WRAP() \
double post_wrap = darshan_core_wtime(); \
psx_wrap_time += (post_wrap-pre_wrap);
/**********************************************************
* Wrappers for POSIX I/O functions of interest *
**********************************************************/
......@@ -356,6 +366,8 @@ int DARSHAN_DECL(open)(const char *path, int flags, ...)
int ret;
double tm1, tm2;
PSX_PRE_WRAP();
MAP_OR_FAIL(open);
if(flags & O_CREAT)
......@@ -381,6 +393,8 @@ int DARSHAN_DECL(open)(const char *path, int flags, ...)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
POSIX_UNLOCK();
PSX_POST_WRAP();
return(ret);
}
......@@ -390,6 +404,8 @@ int DARSHAN_DECL(open64)(const char *path, int flags, ...)
int ret;
double tm1, tm2;
PSX_PRE_WRAP();
MAP_OR_FAIL(open64);
if(flags & O_CREAT)
......@@ -415,6 +431,8 @@ int DARSHAN_DECL(open64)(const char *path, int flags, ...)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
POSIX_UNLOCK();
PSX_POST_WRAP();
return(ret);
}
......@@ -423,6 +441,8 @@ int DARSHAN_DECL(creat)(const char* path, mode_t mode)
int ret;
double tm1, tm2;
PSX_PRE_WRAP();
MAP_OR_FAIL(creat);
tm1 = darshan_core_wtime();
......@@ -434,6 +454,8 @@ int DARSHAN_DECL(creat)(const char* path, mode_t mode)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
POSIX_UNLOCK();
PSX_POST_WRAP();
return(ret);
}
......@@ -442,6 +464,8 @@ int DARSHAN_DECL(creat64)(const char* path, mode_t mode)
int ret;
double tm1, tm2;
PSX_PRE_WRAP();
MAP_OR_FAIL(creat64);
tm1 = darshan_core_wtime();
......@@ -453,6 +477,8 @@ int DARSHAN_DECL(creat64)(const char* path, mode_t mode)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);