Commit 34c54d07 authored by Huihuo Zheng's avatar Huihuo Zheng
Browse files

fixed some path issues

parent dd76ada7
......@@ -440,9 +440,11 @@ H5VL_pass_through_ext_register(void)
static herr_t
H5VL_pass_through_ext_init(hid_t vipl_id)
{
printf("%s:%d: Pass_through VOL is called.\n", __func__, __LINE__);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank==io_node()) printf("%s:%d: Pass_through VOL is called.\n", __func__, __LINE__);
#ifdef ENABLE_EXT_PASSTHRU_LOGGING
printf("------- EXT PASS THROUGH VOL INIT\n");
if (rank==io_node()) printf("------- EXT PASS THROUGH VOL INIT\n");
#endif
/* Shut compiler up about unused parameter */
......@@ -1245,8 +1247,8 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
msync(task->buf, task->size, MS_SYNC);
H5VL_pass_through_ext_t *o = (H5VL_pass_through_ext_t *)task->dataset_obj;
hbool_t acquired=false;
//while(!acquired)
//H5TSmutex_acquire(&acquired);
while(!acquired)
H5TSmutex_acquire(&acquired);
if (wmm->mpi.rank== io_node() && debug_level()>1) printf("pthread: acquired global mutex\n");
H5VLrestore_lib_state(task->h5_state);
void **req;
......@@ -1272,7 +1274,7 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
H5Tclose(task->mem_type_id);
H5VLreset_lib_state();
H5VLfree_lib_state(task->h5_state);
//H5TSmutex_release();
H5TSmutex_release();
if (wmm->mpi.rank==io_node() && debug_level()>1) printf("pthread: global mutex_released\n");
if (wmm->mpi.rank== io_node() && debug_level()>1) {
printf("===================================\n");
......@@ -1293,7 +1295,6 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
pthread_cond_wait(&wmm->io.io_cond, &wmm->io.request_lock);
pthread_mutex_unlock(&wmm->io.request_lock);
}
pthread_mutex_lock(&wmm->io.request_lock);
loop = (wmm->io.num_request>=0);
empty = (wmm->io.num_request==0);
......@@ -1765,6 +1766,20 @@ H5VL_pass_through_ext_datatype_close(void *dt, hid_t dxpl_id, void **req)
return ret_value;
} /* end H5VL_pass_through_ext_datatype_close() */
char *get_fname(const char *path) {
char *ssc;
int l = 0;
char *pp = (char *) malloc(255);
strcpy(pp, path);
ssc = strstr(pp, "/");
do {
l = strlen(ssc) + 1;
pp= &pp[strlen(path)-l+2];
ssc = strstr(pp, "/");
} while(ssc);
return pp;
}
/*-------------------------------------------------------------------------
* Function: H5VL_pass_through_ext_file_create
......@@ -1837,20 +1852,22 @@ H5VL_pass_through_ext_file_create(const char *name, unsigned flags, hid_t fcpl_i
MPI_Comm_rank(file->H5DWMM->mpi.node_comm, &file->H5DWMM->mpi.local_rank);
MPI_Comm_size(file->H5DWMM->mpi.node_comm, &file->H5DWMM->mpi.ppn);
strcpy(file->H5DWMM->mmap.fname, file->H5DWMM->ssd->path);
strcpy(file->H5DWMM->mmap.fname, name);
strcat(file->H5DWMM->mmap.fname, "-");
strcat(file->H5DWMM->mmap.fname, "/cache-");
//strcat(file->H5DWMM->mmap.fname, get_fname(name));
char rnd[255];
sprintf(rnd, "%d", rand());
strcat(file->H5DWMM->mmap.fname, rnd);
strcat(file->H5DWMM->mmap.fname, "-");
sprintf(rnd, "%d", file->H5DWMM->mpi.rank);
strcat(file->H5DWMM->mmap.fname, rnd);
strcat(file->H5DWMM->mmap.fname, "_mmapf.dat");
file->H5DWMM->io.request_list = (thread_data_t*) malloc(sizeof(thread_data_t));
file->H5DWMM->ssd->mspace_per_rank_total = file->H5DWMM->ssd->mspace_total / file->H5DWMM->mpi.ppn;
file->H5DWMM->ssd->mspace_per_rank_left = file->H5DWMM->ssd->mspace_per_rank_total;
if (debug_level()>1 && io_node()==file->H5DWMM->mpi.rank) {
printf("**Using node local storage as a cache\n");
printf("**path: %s\n", file->H5DWMM->ssd->path);
printf("**fname: %20s\n", file->H5DWMM->mmap.fname);
}
file->H5DWMM->mmap.fd = open(file->H5DWMM->mmap.fname, O_RDWR | O_CREAT | O_TRUNC, 0644);
int rc = pthread_create(&file->H5DWMM->io.pthread, NULL, H5Dwrite_pthread_func_vol, file->H5DWMM);
......@@ -2163,8 +2180,11 @@ H5VL_pass_through_ext_file_close(void *file, hid_t dxpl_id, void **req)
pthread_cond_signal(&o->H5DWMM->io.io_cond);
pthread_mutex_unlock(&o->H5DWMM->io.request_lock);
pthread_join(o->H5DWMM->io.pthread, NULL);
double t0 = MPI_Wtime();
close(o->H5DWMM->mmap.fd);
remove(o->H5DWMM->mmap.fname);
double t1 = MPI_Wtime();
if (o->H5DWMM->mpi.rank==io_node() && debug_level()>1) printf(" *** Remove memory mapped files: %f seconds \n", t1 - t0);
o->H5DWMM->ssd->mspace_left = o->H5DWMM->ssd->mspace_total;
hbool_t acq = false;
while(!acq)
......
......@@ -12,8 +12,8 @@ LIBS=-L$(HDF5_DIR)/lib -lhdf5 -lz
CFLAGS=$(INCLUDES) $(DEBUG)
#TARGET=libh5passthrough_vol.so
TARGET=libh5passthrough_vol.dylib
TARGET=libh5passthrough_vol.so
#TARGET=libh5passthrough_vol.dylib
all: makeso test_write_cache
......
......@@ -54,6 +54,7 @@ int main(int argc, char **argv) {
int niter = 10;
char scratch[255] = "./";
double sleep=0.0;
bool collective=false;
for(int i=1; i<argc; i++) {
if (strcmp(argv[i], "--dim")==0) {
d1 = int(atoi(argv[i+1]));
......@@ -68,6 +69,8 @@ int main(int argc, char **argv) {
} else if (strcmp(argv[i],"--sleep")==0) {
sleep = atof(argv[i+1]);
i+=1;
} else if (strcmp(argv[i], "--collective")==0) {
collective = true;
}
}
hsize_t ldims[2] = {d1, d2};
......@@ -101,7 +104,7 @@ int main(int argc, char **argv) {
char f[255];
strcpy(f, scratch);
strcat(f, "/parallel_file.h5");
strcat(f, "./parallel_file.h5");
tt.start_clock("H5Fcreate");
hid_t file_id = H5Fcreate(f, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
tt.stop_clock("H5Fcreate");
......@@ -112,7 +115,10 @@ int main(int argc, char **argv) {
int* data = new int[ldims[0]*ldims[1]*2];
// set up dataset access property list
hid_t dxf_id = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_COLLECTIVE);
if (collective)
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_COLLECTIVE);
else
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_INDEPENDENT);
hsize_t ggdims[2] = {gdims[0]*niter, gdims[1]};
hid_t filespace = H5Screate_simple(2, ggdims, NULL);
......@@ -154,9 +160,12 @@ int main(int argc, char **argv) {
H5Sselect_hyperslab(memspace, H5S_SELECT_OR, offset, NULL, loc_buf_select, count);
tt.stop_clock("Select");
if (rank==0 and i==0) printf(" Selected buffer size (Bytes): %llu (memspace) - %llu (filespace) \n", get_buf_size(memspace, H5T_NATIVE_INT), get_buf_size(filespace, H5T_NATIVE_INT));
MPI_Barrier(MPI_COMM_WORLD);
tt.start_clock("H5Dwrite");
hid_t status = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, dxf_id, data); // write memory to file
tt.stop_clock("H5Dwrite");
if (rank==0) printf(" *Iter: %d - write rate: %f MiB/s\n", i, get_buf_size(memspace, H5T_NATIVE_INT)/tt["H5Dwrite"].t_iter[i]*nproc/1024/1024);
tt.start_clock("compute");
msleep(int(sleep*1000));
tt.stop_clock("compute");
......@@ -176,7 +185,7 @@ int main(int argc, char **argv) {
double avg = 0.0;
double std = 0.0;
stat(&T.t_iter[0], niter, avg, std, 'i');
if (rank==0) printf(" Write rate: %f +/- %f MB/s\n", size*avg/niter/1024/1024, size/niter*std/1024/1024);
if (rank==0) printf("Overall write rate: %f +/- %f MB/s\n", size*avg/niter/1024/1024, size/niter*std/1024/1024);
tt.stop_clock("total");
MPI_Finalize();
return 0;
......
......@@ -28,7 +28,7 @@
using namespace std;
int main(int argc, char **argv) {
int dim = 1024576;
int nbatch = 1024;
int nbatch = 32;
int niter = 10;
double dat[dim];
double vm, rss;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment