Commit e7988689 authored by Richard Zamora's avatar Richard Zamora
Browse files

adding option --rshift to require the read to occur on data that is shifted...

adding option --rshift to require the read to occur on data that is shifted from position where the rank originally wrote to.  By setting this to ppn, the option should allow us to measure the read performance with mitigated caching effects
parent db6b1a16
......@@ -127,6 +127,7 @@ extern void set_MPI_File_write_at(int setval);
#define MED_ind 3
#define MAX_ind 4
#define doRead 1
#define do_immediate_read 0 // <-- Read back without closing the file first
void getmoments(double *ain, int n, double *momarr);
static int CountTasksPerNode(int numTasks, int rank, MPI_Comm comm);
......@@ -163,6 +164,7 @@ int main( int argc, char* argv[] )
int useChunked = 0;
int usemem = 0;
int topohint = 0;
int rankshift = 0;
double memadd = 0.0;
int maxcheck_set = 0;
// defaults are 1
......@@ -176,6 +178,7 @@ int main( int argc, char* argv[] )
int curNEls[MAX_DIM];
int dimRanks[MAX_DIM];
int rankLocation[MAX_DIM];
int rankLocationShifted[MAX_DIM];
// Mem/File space settings
hsize_t fileBlock_def[MAX_DIM];
......@@ -192,6 +195,7 @@ int main( int argc, char* argv[] )
curNEls[i] = 8;
dimRanks[i] = 1;
rankLocation[i] = 1;
rankLocationShifted[i] = 1;
fileBlock_def[i] = 0;
fileBlock_dbl[i] = 0;
fileBlock_spe[i] = 0;
......@@ -295,6 +299,9 @@ int main( int argc, char* argv[] )
i++; dimRanks[j] = atoi(argv[i]);
}
}
else if (strcmp(argv[i],"--rshift") == 0) {
i++; rankshift = atoi(argv[i]);
}
else {
printf("ERROR - unrecognized parameter: %s. Exitting.\n",argv[i]);
exit(-1);
......@@ -339,11 +346,21 @@ int main( int argc, char* argv[] )
rankLocation[i] = cary % dimRanks[i];
cary = cary / dimRanks[i];
}
// If we are shifting the ranks...
if (rankshift > 0) {
// We are basically shifting the location of the ranks in the file...
cary = rank + rankshift;
if (cary >= nprocs) cary -= nprocs;
for(i=numDims-1; i >= 0; --i){
rankLocationShifted[i] = cary % dimRanks[i];
cary = cary / dimRanks[i];
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("useMetaDataCollectives: %d addDerivedTypeDataset: %d addAttributes: %d useIndependentIO: %d numDims: %d useChunked: %d\n",
useMetaDataCollectives,addDerivedTypeDataset,addAttributes,useIndependentIO,numDims,useChunked);
printf("useMetaDataCollectives: %d addDerivedTypeDataset: %d addAttributes: %d useIndependentIO: %d numDims: %d useChunked: %d readShift: %d\n",
useMetaDataCollectives,addDerivedTypeDataset,addAttributes,useIndependentIO,numDims,useChunked,rankshift);
printf("Metric Bufsize H5DWrite RawWrBDWTH H5Dread RawRdBDWTH Dataset Group Attribute H5Fopen H5Fclose H5Fflush OtherClose\n");
printf("WARNING: Bufsize is the size of the double-type dataset (The derived-type dataset is 8x larger!).\n");
}
......@@ -495,6 +512,10 @@ int main( int argc, char* argv[] )
// data buffers for writing and reading
double *dataBuffer = (double *) malloc(sizeof(double) * NumDoubleElements);
double *checkBuffer = (double *) malloc(sizeof(double) * NumDoubleElements);
double *dataBufferShift;
if (rankshift > 0) {
dataBufferShift = (double *) malloc(sizeof(double) * NumDoubleElements);
}
sim_params_t *sim_params = (sim_params_t *) malloc(sizeof(sim_params_t) * NumSimParamElements);
for (i=0;i<NumSimParamElements;i++) {
sim_params[i].total_blocks = i+rank*100;
......@@ -506,7 +527,7 @@ int main( int argc, char* argv[] )
sim_params[i].nzb = i+rank*100;
}
// init with some data
// init buffers with some data
for (i=0;i<memCount_dbl;i++){
for (j=0;j<memStride_dbl;j++){
int ii = i * memStride_dbl + j;
......@@ -517,6 +538,16 @@ int main( int argc, char* argv[] )
dataBuffer[ii] = 0.0;
}
checkBuffer[ii] = 0.0;
// Populate a buffer with expected data for "shifted" read...
if (rankshift > 0) {
if (j < memBlock_dbl){
cary = rank + rankshift;
if (cary >= nprocs) cary -= nprocs;
dataBufferShift[ii] = (double)(cary + 1);
}else{
dataBufferShift[ii] = 0.0;
}
}
}
}
......@@ -737,21 +768,25 @@ int main( int argc, char* argv[] )
}
#endif
// read to check the write results
MPI_Barrier(comm);
startTime = MPI_Wtime();
if (doRead && doWriteat) H5Dread(dataSet, H5T_NATIVE_DOUBLE, memDataSpace, fileDataSpace, xferPropList, checkBuffer);
readDataTime += (MPI_Wtime()-startTime);
if (do_immediate_read) {
// verify the data only for smaller buffer sizes
if (doWriteat) {
if (!maxcheck_set || (BufSizeTotalDouble <= maxcheck)) {
for (i=0;i<NumDoubleElements;i++) {
if (dataBuffer[i] != checkBuffer[i]) {
printf("ERROR on read: index %d doesn't match - expected %20.16f got %20.16f \n",i,dataBuffer[i],checkBuffer[i]);
// read to check the write results
MPI_Barrier(comm);
startTime = MPI_Wtime();
if (doRead && doWriteat) H5Dread(dataSet, H5T_NATIVE_DOUBLE, memDataSpace, fileDataSpace, xferPropList, checkBuffer);
readDataTime += (MPI_Wtime()-startTime);
// verify the data only for smaller buffer sizes
if (doWriteat) {
if (!maxcheck_set || (BufSizeTotalDouble <= maxcheck)) {
for (i=0;i<NumDoubleElements;i++) {
if (dataBuffer[i] != checkBuffer[i]) {
printf("ERROR on read: index %d doesn't match - expected %20.16f got %20.16f \n",i,dataBuffer[i],checkBuffer[i]);
}
}
}
}
}
// close everything and re-read it
......@@ -836,6 +871,15 @@ int main( int argc, char* argv[] )
dtype_id = H5Dget_type(dataSet);
plist_id = H5Dget_create_plist(dataSet);
// Change rankLocation list if we are shifting the ranks...
if (rankshift > 0) {
// We are basically shifting the location of the ranks in the file...
for (i=0;i<numDims;i++) {
fileStart[i] = fileStride[i] * fileCount[i] * rankLocationShifted[i];
}
MPI_Barrier(MPI_COMM_WORLD);
}
/* define hyperslab for memory data space */
H5Sselect_hyperslab(memDataSpace, H5S_SELECT_SET, memStart, memStride, memCount, memBlock);
......@@ -844,7 +888,7 @@ int main( int argc, char* argv[] )
MPI_Barrier(comm);
startTime = MPI_Wtime();
if (doRead && doWriteat) H5Dread(dataSet, H5T_NATIVE_DOUBLE, memDataSpace, fileDataSpace,xferPropList, checkBuffer);
if (doRead && doWriteat) H5Dread(dataSet, H5T_NATIVE_DOUBLE, memDataSpace, fileDataSpace, xferPropList, checkBuffer);
timenow = MPI_Wtime();
readDataTime += (timenow-startTime);
if((timenow-startTime)>0.0){
......@@ -859,8 +903,10 @@ int main( int argc, char* argv[] )
if (doWriteat) {
if (!maxcheck_set || (BufSizeTotalDouble <= maxcheck)) {
for (i=0;i<(NumDoubleElements);i++) {
if (dataBuffer[i] != checkBuffer[i]) {
if (rankshift == 0 && (dataBuffer[i] != checkBuffer[i])) {
printf("ERROR on read: index %d doesn't match - expected %20.16f got %20.16f \n",i,dataBuffer[i],checkBuffer[i]);
} else if (rankshift > 0 && (dataBufferShift[i] != checkBuffer[i])) {
printf("ERROR on read: index %d doesn't match - expected %20.16f got %20.16f \n",i,dataBufferShift[i],checkBuffer[i]);
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment