Commit 34c54d07 authored by Huihuo Zheng's avatar Huihuo Zheng
Browse files

fixed some path issues

parent dd76ada7
...@@ -440,9 +440,11 @@ H5VL_pass_through_ext_register(void) ...@@ -440,9 +440,11 @@ H5VL_pass_through_ext_register(void)
static herr_t static herr_t
H5VL_pass_through_ext_init(hid_t vipl_id) H5VL_pass_through_ext_init(hid_t vipl_id)
{ {
printf("%s:%d: Pass_through VOL is called.\n", __func__, __LINE__); int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (rank==io_node()) printf("%s:%d: Pass_through VOL is called.\n", __func__, __LINE__);
#ifdef ENABLE_EXT_PASSTHRU_LOGGING #ifdef ENABLE_EXT_PASSTHRU_LOGGING
printf("------- EXT PASS THROUGH VOL INIT\n"); if (rank==io_node()) printf("------- EXT PASS THROUGH VOL INIT\n");
#endif #endif
/* Shut compiler up about unused parameter */ /* Shut compiler up about unused parameter */
...@@ -1245,8 +1247,8 @@ void *H5Dwrite_pthread_func_vol(void *arg) { ...@@ -1245,8 +1247,8 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
msync(task->buf, task->size, MS_SYNC); msync(task->buf, task->size, MS_SYNC);
H5VL_pass_through_ext_t *o = (H5VL_pass_through_ext_t *)task->dataset_obj; H5VL_pass_through_ext_t *o = (H5VL_pass_through_ext_t *)task->dataset_obj;
hbool_t acquired=false; hbool_t acquired=false;
//while(!acquired) while(!acquired)
//H5TSmutex_acquire(&acquired); H5TSmutex_acquire(&acquired);
if (wmm->mpi.rank== io_node() && debug_level()>1) printf("pthread: acquired global mutex\n"); if (wmm->mpi.rank== io_node() && debug_level()>1) printf("pthread: acquired global mutex\n");
H5VLrestore_lib_state(task->h5_state); H5VLrestore_lib_state(task->h5_state);
void **req; void **req;
...@@ -1272,7 +1274,7 @@ void *H5Dwrite_pthread_func_vol(void *arg) { ...@@ -1272,7 +1274,7 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
H5Tclose(task->mem_type_id); H5Tclose(task->mem_type_id);
H5VLreset_lib_state(); H5VLreset_lib_state();
H5VLfree_lib_state(task->h5_state); H5VLfree_lib_state(task->h5_state);
//H5TSmutex_release(); H5TSmutex_release();
if (wmm->mpi.rank==io_node() && debug_level()>1) printf("pthread: global mutex_released\n"); if (wmm->mpi.rank==io_node() && debug_level()>1) printf("pthread: global mutex_released\n");
if (wmm->mpi.rank== io_node() && debug_level()>1) { if (wmm->mpi.rank== io_node() && debug_level()>1) {
printf("===================================\n"); printf("===================================\n");
...@@ -1293,7 +1295,6 @@ void *H5Dwrite_pthread_func_vol(void *arg) { ...@@ -1293,7 +1295,6 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
pthread_cond_wait(&wmm->io.io_cond, &wmm->io.request_lock); pthread_cond_wait(&wmm->io.io_cond, &wmm->io.request_lock);
pthread_mutex_unlock(&wmm->io.request_lock); pthread_mutex_unlock(&wmm->io.request_lock);
} }
pthread_mutex_lock(&wmm->io.request_lock); pthread_mutex_lock(&wmm->io.request_lock);
loop = (wmm->io.num_request>=0); loop = (wmm->io.num_request>=0);
empty = (wmm->io.num_request==0); empty = (wmm->io.num_request==0);
...@@ -1765,6 +1766,20 @@ H5VL_pass_through_ext_datatype_close(void *dt, hid_t dxpl_id, void **req) ...@@ -1765,6 +1766,20 @@ H5VL_pass_through_ext_datatype_close(void *dt, hid_t dxpl_id, void **req)
return ret_value; return ret_value;
} /* end H5VL_pass_through_ext_datatype_close() */ } /* end H5VL_pass_through_ext_datatype_close() */
char *get_fname(const char *path) {
char *ssc;
int l = 0;
char *pp = (char *) malloc(255);
strcpy(pp, path);
ssc = strstr(pp, "/");
do {
l = strlen(ssc) + 1;
pp= &pp[strlen(path)-l+2];
ssc = strstr(pp, "/");
} while(ssc);
return pp;
}
/*------------------------------------------------------------------------- /*-------------------------------------------------------------------------
* Function: H5VL_pass_through_ext_file_create * Function: H5VL_pass_through_ext_file_create
...@@ -1837,20 +1852,22 @@ H5VL_pass_through_ext_file_create(const char *name, unsigned flags, hid_t fcpl_i ...@@ -1837,20 +1852,22 @@ H5VL_pass_through_ext_file_create(const char *name, unsigned flags, hid_t fcpl_i
MPI_Comm_rank(file->H5DWMM->mpi.node_comm, &file->H5DWMM->mpi.local_rank); MPI_Comm_rank(file->H5DWMM->mpi.node_comm, &file->H5DWMM->mpi.local_rank);
MPI_Comm_size(file->H5DWMM->mpi.node_comm, &file->H5DWMM->mpi.ppn); MPI_Comm_size(file->H5DWMM->mpi.node_comm, &file->H5DWMM->mpi.ppn);
strcpy(file->H5DWMM->mmap.fname, file->H5DWMM->ssd->path); strcpy(file->H5DWMM->mmap.fname, file->H5DWMM->ssd->path);
strcpy(file->H5DWMM->mmap.fname, name); strcat(file->H5DWMM->mmap.fname, "/cache-");
strcat(file->H5DWMM->mmap.fname, "-"); //strcat(file->H5DWMM->mmap.fname, get_fname(name));
char rnd[255]; char rnd[255];
sprintf(rnd, "%d", rand()); sprintf(rnd, "%d", rand());
strcat(file->H5DWMM->mmap.fname, rnd); strcat(file->H5DWMM->mmap.fname, rnd);
strcat(file->H5DWMM->mmap.fname, "-"); strcat(file->H5DWMM->mmap.fname, "-");
sprintf(rnd, "%d", file->H5DWMM->mpi.rank); sprintf(rnd, "%d", file->H5DWMM->mpi.rank);
strcat(file->H5DWMM->mmap.fname, rnd); strcat(file->H5DWMM->mmap.fname, rnd);
strcat(file->H5DWMM->mmap.fname, "_mmapf.dat");
file->H5DWMM->io.request_list = (thread_data_t*) malloc(sizeof(thread_data_t)); file->H5DWMM->io.request_list = (thread_data_t*) malloc(sizeof(thread_data_t));
file->H5DWMM->ssd->mspace_per_rank_total = file->H5DWMM->ssd->mspace_total / file->H5DWMM->mpi.ppn; file->H5DWMM->ssd->mspace_per_rank_total = file->H5DWMM->ssd->mspace_total / file->H5DWMM->mpi.ppn;
file->H5DWMM->ssd->mspace_per_rank_left = file->H5DWMM->ssd->mspace_per_rank_total; file->H5DWMM->ssd->mspace_per_rank_left = file->H5DWMM->ssd->mspace_per_rank_total;
if (debug_level()>1 && io_node()==file->H5DWMM->mpi.rank) { if (debug_level()>1 && io_node()==file->H5DWMM->mpi.rank) {
printf("**Using node local storage as a cache\n"); printf("**Using node local storage as a cache\n");
printf("**path: %s\n", file->H5DWMM->ssd->path); printf("**path: %s\n", file->H5DWMM->ssd->path);
printf("**fname: %20s\n", file->H5DWMM->mmap.fname);
} }
file->H5DWMM->mmap.fd = open(file->H5DWMM->mmap.fname, O_RDWR | O_CREAT | O_TRUNC, 0644); file->H5DWMM->mmap.fd = open(file->H5DWMM->mmap.fname, O_RDWR | O_CREAT | O_TRUNC, 0644);
int rc = pthread_create(&file->H5DWMM->io.pthread, NULL, H5Dwrite_pthread_func_vol, file->H5DWMM); int rc = pthread_create(&file->H5DWMM->io.pthread, NULL, H5Dwrite_pthread_func_vol, file->H5DWMM);
...@@ -2163,8 +2180,11 @@ H5VL_pass_through_ext_file_close(void *file, hid_t dxpl_id, void **req) ...@@ -2163,8 +2180,11 @@ H5VL_pass_through_ext_file_close(void *file, hid_t dxpl_id, void **req)
pthread_cond_signal(&o->H5DWMM->io.io_cond); pthread_cond_signal(&o->H5DWMM->io.io_cond);
pthread_mutex_unlock(&o->H5DWMM->io.request_lock); pthread_mutex_unlock(&o->H5DWMM->io.request_lock);
pthread_join(o->H5DWMM->io.pthread, NULL); pthread_join(o->H5DWMM->io.pthread, NULL);
double t0 = MPI_Wtime();
close(o->H5DWMM->mmap.fd); close(o->H5DWMM->mmap.fd);
remove(o->H5DWMM->mmap.fname); remove(o->H5DWMM->mmap.fname);
double t1 = MPI_Wtime();
if (o->H5DWMM->mpi.rank==io_node() && debug_level()>1) printf(" *** Remove memory mapped files: %f seconds \n", t1 - t0);
o->H5DWMM->ssd->mspace_left = o->H5DWMM->ssd->mspace_total; o->H5DWMM->ssd->mspace_left = o->H5DWMM->ssd->mspace_total;
hbool_t acq = false; hbool_t acq = false;
while(!acq) while(!acq)
......
...@@ -12,8 +12,8 @@ LIBS=-L$(HDF5_DIR)/lib -lhdf5 -lz ...@@ -12,8 +12,8 @@ LIBS=-L$(HDF5_DIR)/lib -lhdf5 -lz
CFLAGS=$(INCLUDES) $(DEBUG) CFLAGS=$(INCLUDES) $(DEBUG)
#TARGET=libh5passthrough_vol.so TARGET=libh5passthrough_vol.so
TARGET=libh5passthrough_vol.dylib #TARGET=libh5passthrough_vol.dylib
all: makeso test_write_cache all: makeso test_write_cache
......
...@@ -54,6 +54,7 @@ int main(int argc, char **argv) { ...@@ -54,6 +54,7 @@ int main(int argc, char **argv) {
int niter = 10; int niter = 10;
char scratch[255] = "./"; char scratch[255] = "./";
double sleep=0.0; double sleep=0.0;
bool collective=false;
for(int i=1; i<argc; i++) { for(int i=1; i<argc; i++) {
if (strcmp(argv[i], "--dim")==0) { if (strcmp(argv[i], "--dim")==0) {
d1 = int(atoi(argv[i+1])); d1 = int(atoi(argv[i+1]));
...@@ -68,6 +69,8 @@ int main(int argc, char **argv) { ...@@ -68,6 +69,8 @@ int main(int argc, char **argv) {
} else if (strcmp(argv[i],"--sleep")==0) { } else if (strcmp(argv[i],"--sleep")==0) {
sleep = atof(argv[i+1]); sleep = atof(argv[i+1]);
i+=1; i+=1;
} else if (strcmp(argv[i], "--collective")==0) {
collective = true;
} }
} }
hsize_t ldims[2] = {d1, d2}; hsize_t ldims[2] = {d1, d2};
...@@ -101,7 +104,7 @@ int main(int argc, char **argv) { ...@@ -101,7 +104,7 @@ int main(int argc, char **argv) {
char f[255]; char f[255];
strcpy(f, scratch); strcpy(f, scratch);
strcat(f, "/parallel_file.h5"); strcat(f, "./parallel_file.h5");
tt.start_clock("H5Fcreate"); tt.start_clock("H5Fcreate");
hid_t file_id = H5Fcreate(f, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); hid_t file_id = H5Fcreate(f, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
tt.stop_clock("H5Fcreate"); tt.stop_clock("H5Fcreate");
...@@ -112,7 +115,10 @@ int main(int argc, char **argv) { ...@@ -112,7 +115,10 @@ int main(int argc, char **argv) {
int* data = new int[ldims[0]*ldims[1]*2]; int* data = new int[ldims[0]*ldims[1]*2];
// set up dataset access property list // set up dataset access property list
hid_t dxf_id = H5Pcreate(H5P_DATASET_XFER); hid_t dxf_id = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_COLLECTIVE); if (collective)
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_COLLECTIVE);
else
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_INDEPENDENT);
hsize_t ggdims[2] = {gdims[0]*niter, gdims[1]}; hsize_t ggdims[2] = {gdims[0]*niter, gdims[1]};
hid_t filespace = H5Screate_simple(2, ggdims, NULL); hid_t filespace = H5Screate_simple(2, ggdims, NULL);
...@@ -154,9 +160,12 @@ int main(int argc, char **argv) { ...@@ -154,9 +160,12 @@ int main(int argc, char **argv) {
H5Sselect_hyperslab(memspace, H5S_SELECT_OR, offset, NULL, loc_buf_select, count); H5Sselect_hyperslab(memspace, H5S_SELECT_OR, offset, NULL, loc_buf_select, count);
tt.stop_clock("Select"); tt.stop_clock("Select");
if (rank==0 and i==0) printf(" Selected buffer size (Bytes): %llu (memspace) - %llu (filespace) \n", get_buf_size(memspace, H5T_NATIVE_INT), get_buf_size(filespace, H5T_NATIVE_INT)); if (rank==0 and i==0) printf(" Selected buffer size (Bytes): %llu (memspace) - %llu (filespace) \n", get_buf_size(memspace, H5T_NATIVE_INT), get_buf_size(filespace, H5T_NATIVE_INT));
MPI_Barrier(MPI_COMM_WORLD);
tt.start_clock("H5Dwrite"); tt.start_clock("H5Dwrite");
hid_t status = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, dxf_id, data); // write memory to file hid_t status = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, dxf_id, data); // write memory to file
tt.stop_clock("H5Dwrite"); tt.stop_clock("H5Dwrite");
if (rank==0) printf(" *Iter: %d - write rate: %f MiB/s\n", i, get_buf_size(memspace, H5T_NATIVE_INT)/tt["H5Dwrite"].t_iter[i]*nproc/1024/1024);
tt.start_clock("compute"); tt.start_clock("compute");
msleep(int(sleep*1000)); msleep(int(sleep*1000));
tt.stop_clock("compute"); tt.stop_clock("compute");
...@@ -176,7 +185,7 @@ int main(int argc, char **argv) { ...@@ -176,7 +185,7 @@ int main(int argc, char **argv) {
double avg = 0.0; double avg = 0.0;
double std = 0.0; double std = 0.0;
stat(&T.t_iter[0], niter, avg, std, 'i'); stat(&T.t_iter[0], niter, avg, std, 'i');
if (rank==0) printf(" Write rate: %f +/- %f MB/s\n", size*avg/niter/1024/1024, size/niter*std/1024/1024); if (rank==0) printf("Overall write rate: %f +/- %f MB/s\n", size*avg/niter/1024/1024, size/niter*std/1024/1024);
tt.stop_clock("total"); tt.stop_clock("total");
MPI_Finalize(); MPI_Finalize();
return 0; return 0;
......
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
using namespace std; using namespace std;
int main(int argc, char **argv) { int main(int argc, char **argv) {
int dim = 1024576; int dim = 1024576;
int nbatch = 1024; int nbatch = 32;
int niter = 10; int niter = 10;
double dat[dim]; double dat[dim];
double vm, rss; double vm, rss;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment