Commit 0836db74 authored by Huihuo Zheng's avatar Huihuo Zheng
Browse files

non contiguous memspace

parent b143cee1
......@@ -87,18 +87,12 @@ void int2char(int a, char str[255]) {
Output: the size of the memory space in bytes.
*/
hsize_t get_buf_size(hid_t mspace, hid_t tid) {
int n= H5Sget_simple_extent_ndims(mspace);
hsize_t *dim = (hsize_t *) malloc(sizeof(hsize_t)*n);
hsize_t *mdim = (hsize_t *) malloc(sizeof(hsize_t)*n);
H5Sget_simple_extent_dims(mspace, dim, mdim);
hsize_t nelement = H5Sget_select_npoints(mspace);
hsize_t s = H5Tget_size(tid);
for(int i=0; i<n; i++)
s=s*dim[i];
free(dim);
free(mdim);
return s;
return s*nelement;
}
/*
Thread function for performing H5Dwrite. This function will create
a memory mapped buffer to the file that is on the local storage which
......
......@@ -1237,12 +1237,13 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
while (loop) {
if (!empty) {
thread_data_t *task = wmm->io.current_request;
msync(task->buf, task->size, MS_SYNC);
if (wmm->mpi.rank== io_node() && debug_level()>1) {
printf("\n===================================\n");
printf("pthread: Executing I/O task %d\n", task->id);
}
task->buf = mmap(NULL, task->size, PROT_READ, MAP_SHARED, wmm->mmap.fd, task->offset);
msync(task->buf, task->size, MS_SYNC);
H5VL_pass_through_ext_t *o = (H5VL_pass_through_ext_t *)task->dataset_obj;
hbool_t acquired=false;
//while(!acquired)
......@@ -1313,6 +1314,25 @@ void *H5Dwrite_pthread_func_vol(void *arg) {
*
*-------------------------------------------------------------------------
*/
herr_t H5Ssel_gather_write(hid_t space, hid_t tid, const void *buf, int fd, hsize_t offset) {
unsigned flags;
size_t elmt_size = H5Tget_size(tid);
hid_t iter = H5Ssel_iter_create(space, elmt_size, flags);
size_t maxseq = H5Sget_select_npoints(space);
size_t maxbytes = maxseq*elmt_size;
size_t nseq, nbytes;
size_t *len = (size_t*)malloc(maxseq*sizeof(size_t));
hsize_t *off = (hsize_t*)malloc(maxseq*sizeof(hsize_t));
H5Ssel_iter_get_seq_list(iter, maxseq, maxbytes, &nseq, &nbytes, off, len);
hsize_t off_contig=0;
for(int i=0; i<nseq; i++) {
// printf("nseq: %d\n", nseq);
int err = pwrite(fd, &buf[off[i]], len[i]*elmt_size, offset+off_contig);
off_contig += len[i]*elmt_size;
}
return 0;
}
static herr_t
H5VL_pass_through_ext_dataset_write(void *dset, hid_t mem_type_id, hid_t mem_space_id,
hid_t file_space_id, hid_t plist_id, const void *buf, void **req)
......@@ -1332,13 +1352,11 @@ H5VL_pass_through_ext_dataset_write(void *dset, hid_t mem_type_id, hid_t mem_spa
pthread_cond_wait(&o->H5DWMM->io.master_cond, &o->H5DWMM->io.request_lock);
}
pthread_mutex_unlock(&o->H5DWMM->io.request_lock);
o->H5DWMM->ssd->offset = 0;
o->H5DWMM->ssd->mspace_per_rank_left = o->H5DWMM->ssd->mspace_per_rank_total;
o->H5DWMM->ssd->mspace_left = o->H5DWMM->ssd->mspace_total;
}
int err = pwrite(o->H5DWMM->mmap.fd, (char*)buf, size, o->H5DWMM->ssd->offset);
H5Ssel_gather_write(mem_space_id, mem_type_id, buf, o->H5DWMM->mmap.fd, o->H5DWMM->ssd->offset);
o->H5DWMM->io.request_list->offset = o->H5DWMM->ssd->offset;
o->H5DWMM->ssd->offset += (size/PAGESIZE+1)*PAGESIZE;
o->H5DWMM->ssd->mspace_per_rank_left = o->H5DWMM->ssd->mspace_per_rank_total
......@@ -1348,12 +1366,12 @@ H5VL_pass_through_ext_dataset_write(void *dset, hid_t mem_type_id, hid_t mem_spa
#else
fsync(o->H5DWMM->mmap.fd);
#endif
o->H5DWMM->io.request_list->dataset_obj = dset;
// retrieve current library state;
H5VLretrieve_lib_state(&o->H5DWMM->io.request_list->h5_state);
o->H5DWMM->io.request_list->mem_type_id = H5Tcopy(mem_type_id);
o->H5DWMM->io.request_list->mem_space_id = H5Scopy(mem_space_id);
hsize_t ldims[1] = {H5Sget_select_npoints(mem_space_id)};
o->H5DWMM->io.request_list->mem_space_id = H5Screate_simple(1, ldims, NULL);
o->H5DWMM->io.request_list->file_space_id = H5Scopy(file_space_id);
o->H5DWMM->io.request_list->xfer_plist_id = H5Pcopy(plist_id);
o->H5DWMM->io.request_list->size = size;
......@@ -2132,8 +2150,9 @@ H5VL_pass_through_ext_file_close(void *file, hid_t dxpl_id, void **req)
close(o->H5DWMM->mmap.fd);
remove(o->H5DWMM->mmap.fname);
o->H5DWMM->ssd->mspace_left = o->H5DWMM->ssd->mspace_total;
hbool_t acq;
H5TSmutex_acquire(&acq);
hbool_t acq = false;
while(!acq)
H5TSmutex_acquire(&acq);
}
ret_value = H5VLfile_close(o->under_object, o->under_vol_id, dxpl_id, req);
......
......@@ -30,6 +30,6 @@ makeso: H5VLpassthru_ext.o H5Dio_cache.o ../utils/debug.o
$(CC) -shared $(CFLAGS) $(DEBUG) -o $(TARGET) -fPIC H5VLpassthru_ext.o H5Dio_cache.o ../utils/debug.o $(LIBS)
mv $(TARGET) $(HDF5_DIR)/../vol
clean:
rm -f $(TARGET) *.o parallel_file.h5*
rm -f $(TARGET) *.o parallel_file.h5* test_write_cache
......@@ -109,7 +109,7 @@ int main(int argc, char **argv) {
hid_t memspace = H5Screate_simple(2, ldims, NULL);
// define local data
int* data = new int[ldims[0]*ldims[1]];
int* data = new int[ldims[0]*ldims[1]*2];
// set up dataset access property list
hid_t dxf_id = H5Pcreate(H5P_DATASET_XFER);
H5Pset_dxpl_mpio(dxf_id, H5FD_MPIO_COLLECTIVE);
......@@ -134,14 +134,20 @@ int main(int argc, char **argv) {
for (int i=0; i<niter; i++) {
tt.start_clock("Init_array");
for(int j=0; j<ldims[0]*ldims[1]; j++)
for(int j=0; j<2*ldims[0]*ldims[1]; j++)
data[j] = i;
tt.stop_clock("Init_array");
offset[0]= i*gdims[0] + rank*ldims[0];
// select hyperslab
hid_t filespace = H5Screate_simple(2, ggdims, NULL);
hsize_t ldims[2] = {d1, d2};
hsize_t ldims_select[2] = {d1/2, d2};
hid_t memspace = H5Screate_simple(2, ldims, NULL);
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, ldims, count);
offset[0]=0;
H5Sselect_hyperslab(memspace, H5S_SELECT_SET, offset, NULL, ldims_select, count);
offset[0]=ldims_select[0]*2;
H5Sselect_hyperslab(memspace, H5S_SELECT_OR, offset, NULL, ldims_select, count);
tt.start_clock("H5Dwrite");
hid_t status = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace, filespace, dxf_id, data); // write memory to file
tt.stop_clock("H5Dwrite");
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment