boolbatch_cached;// for parallel read, -- whether the batch data is cached to SSD or not
booldset_cached;// whether the entire dataset is cached to SSD or not.
pthread_tpthread;
}IO_THREAD;
// Memory mapped files
typedefstruct_MMAP{
// for write
intfd;// file handle for write
charfname[255];// full path of the memory mapped file
void*buf;// pointer that map the file to the memory
void*tmp_buf;// temporally buffer, used for parallel read: copy the read buffer, return the H5Dread_to_cache function, the back ground thread write the data to the SSD.
}MMAP;
// Dataset
typedefstruct_SAMPLE{
size_tdim;// the number of dimension
size_tsize;// the size of the sample in bytes.
size_tnel;// the number of elements per sample,
}SAMPLE;
typedefstruct_BATCH{
int*list;
intsize;
}BATCH;
typedefstruct_DSET{
SAMPLEsample;
size_tns_loc;// number of samples per rank
size_tns_glob;// total number of samples
size_ts_offset;// offset
hsize_tsize;// the size of the entire dataset in bytes.
BATCHbatch;// batch data to read
boolcontig_read;// whether the batch of data to read is contigues or not.
MPI_Datatypempi_datatype;// the constructed mpi dataset
hid_th5_datatype;// hdf5 dataset
size_tesize;// the size of an element in bytes.
}DSET;
typedefstruct_H5Dwrite_cache_metadata{
MMAPmmap;
MPI_INFOmpi;
IO_THREADio;
SSD_INFO*ssd;
}H5Dwrite_cache_metadata;
typedefstruct_H5Dread_cache_metadata{
MMAPmmap;
MPI_INFOmpi;
IO_THREADio;
DSETdset;
SSD_INFO*ssd;
}H5Dread_cache_metadata;
/**************************************
* Function APIs for parallel write *
**************************************/
// Create HDF5 file: create memory mapped file on the SSD
