Commit 3e040d8c authored by rjzamora's avatar rjzamora
Browse files

changing rshift to shift both read and write differently - to change pattern and avoid chaching

parent 21091203
......@@ -13,6 +13,7 @@
* ...Each dimension of curNEls is then multiplied by each dimension of bufMult, and the process is repeated.
*
* The exerciser takes the following parameters (all default to 0 or false):
* --rshift: Write to '+rshift' location in file, and read from '+2rshift'
* --topohint: Whether to use topology for aggregator selection and pass MPI hint
* --metacoll: Whether to set meta data collective usage
* --derivedtype: Whether to create a second data set containing a derived type.
......@@ -138,7 +139,6 @@ void getmoments(double *ain, int n, double *momarr);
static int CountTasksPerNode(int numTasks, int rank, MPI_Comm comm);
static void *malloc_and_touch(size_t size);
static size_t NodeMemoryFracToBytes(double frac_d);
double FS_Uncache( MPI_Comm comm );
/* data structure from flashio */
typedef struct sim_params_t {
......@@ -172,7 +172,6 @@ int main( int argc, char* argv[] )
int topohint = 0;
int rankshift = 0;
int dogdb = 0;
int uncache = 0;
double memadd = 0.0;
int maxcheck_set = 0;
// defaults are 1
......@@ -186,7 +185,8 @@ int main( int argc, char* argv[] )
int curNEls[MAX_DIM];
int dimRanks[MAX_DIM];
int rankLocation[MAX_DIM];
int rankLocationShifted[MAX_DIM];
int rankLocationShiftedWr[MAX_DIM];
int rankLocationShiftedRd[MAX_DIM];
// Mem/File space settings
hsize_t fileBlock_def[MAX_DIM];
......@@ -203,7 +203,8 @@ int main( int argc, char* argv[] )
curNEls[i] = 8;
dimRanks[i] = 1;
rankLocation[i] = 1;
rankLocationShifted[i] = 1;
rankLocationShiftedWr[i] = 1;
rankLocationShiftedRd[i] = 1;
fileBlock_def[i] = 0;
fileBlock_dbl[i] = 0;
fileBlock_spe[i] = 0;
......@@ -220,8 +221,6 @@ int main( int argc, char* argv[] )
for (i=1;i<argc;i++) {
if (strcmp(argv[i],"--metacoll") == 0)
useMetaDataCollectives = 1;
else if (strcmp(argv[i],"--uncache") == 0)
uncache = 1;
else if (strcmp(argv[i],"--gdb") == 0)
dogdb = 1;
else if (strcmp(argv[i],"--nowrite") == 0)
......@@ -364,15 +363,22 @@ int main( int argc, char* argv[] )
cary = rank + rankshift;
if (cary >= nprocs) cary -= nprocs;
for(i=numDims-1; i >= 0; --i){
rankLocationShifted[i] = cary % dimRanks[i];
rankLocationShiftedWr[i] = cary % dimRanks[i];
cary = cary / dimRanks[i];
}
// We are basically shifting the location of the ranks in the file...
cary = rank + rankshift * 2;
if (cary >= nprocs) cary -= nprocs;
for(i=numDims-1; i >= 0; --i){
rankLocationShiftedRd[i] = cary % dimRanks[i];
cary = cary / dimRanks[i];
}
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
printf("useMetaDataCollectives: %d addDerivedTypeDataset: %d addAttributes: %d useIndependentIO: %d numDims: %d useChunked: %d rankShift: %d uncache: %d\n",
useMetaDataCollectives,addDerivedTypeDataset,addAttributes,useIndependentIO,numDims,useChunked,rankshift,uncache);
printf("useMetaDataCollectives: %d addDerivedTypeDataset: %d addAttributes: %d useIndependentIO: %d numDims: %d useChunked: %d rankShift: %d\n",
useMetaDataCollectives,addDerivedTypeDataset,addAttributes,useIndependentIO,numDims,useChunked,rankshift);
printf("Metric Bufsize H5DWrite RawWrBDWTH H5Dread RawRdBDWTH Dataset Group Attribute H5Fopen H5Fclose H5Fflush OtherClose\n");
printf("WARNING: Bufsize is the size of the double-type dataset (The derived-type dataset is 8x larger!).\n");
}
......@@ -599,10 +605,10 @@ int main( int argc, char* argv[] )
// data buffers for writing and reading
double *dataBuffer = (double *) malloc(sizeof(double) * NumDoubleElements);
double *checkBuffer = (double *) malloc(sizeof(double) * NumDoubleElements);
//double *dataBufferShift;
//if (0 && (rankshift > 0)) {
// dataBufferShift = (double *) malloc(sizeof(double) * NumDoubleElements);
//}
double *dataBufferShift;
if (rankshift > 0) {
dataBufferShift = (double *) malloc(sizeof(double) * NumDoubleElements);
}
sim_params_t *sim_params = (sim_params_t *) malloc(sizeof(sim_params_t) * NumSimParamElements);
for (i=0;i<NumSimParamElements;i++) {
sim_params[i].total_blocks = i+rank*100;
......@@ -625,6 +631,17 @@ int main( int argc, char* argv[] )
dataBuffer[ii] = 0.0;
}
checkBuffer[ii] = 0.0;
// Populate a buffer with expected data for "shifted" read...
if (rankshift > 0) {
if (j < memBlock_dbl){
cary = rank + rankshift; // * 2;
if (cary >= nprocs) cary -= nprocs;
dataBufferShift[ii] = (double)(cary + 1);
}else{
dataBufferShift[ii] = 0.0;
}
}
}
}
......@@ -787,11 +804,11 @@ int main( int argc, char* argv[] )
memDataSpace = H5Screate_simple(1, memDataSpaceDims, NULL);
fileDataSpace = H5Screate_simple(numDims, dataSetDims, NULL);
// Change rankLocation list if we are shifting the ranks...
// (Write) Change rankLocation list if we are shifting the ranks...
if (rankshift > 0) {
// We are basically shifting the location of the ranks in the file...
for (i=0;i<numDims;i++) {
fileStart[i] = fileStride[i] * fileCount[i] * rankLocationShifted[i];
fileStart[i] = fileStride[i] * fileCount[i] * rankLocationShiftedWr[i];
}
MPI_Barrier(MPI_COMM_WORLD);
}
......@@ -885,10 +902,6 @@ int main( int argc, char* argv[] )
otherCloseTime += (MPI_Wtime()-startTime);
/* Write temp file to dissable caching */
if (uncache)
FS_Uncache(comm);
MPI_Barrier(comm);
startTime = MPI_Wtime();
rc = H5Fflush(fd,H5F_SCOPE_LOCAL);
......@@ -965,7 +978,7 @@ int main( int argc, char* argv[] )
if (rankshift > 0) {
// We are basically shifting the location of the ranks in the file...
for (i=0;i<numDims;i++) {
fileStart[i] = fileStride[i] * fileCount[i] * rankLocationShifted[i];
fileStart[i] = fileStride[i] * fileCount[i] * rankLocationShiftedRd[i];
}
MPI_Barrier(MPI_COMM_WORLD);
}
......@@ -993,8 +1006,10 @@ int main( int argc, char* argv[] )
if (doWriteat) {
if (!maxcheck_set || (BufSizeTotalDouble <= maxcheck)) {
for (i=0;i<(NumDoubleElements);i++) {
if (dataBuffer[i] != checkBuffer[i]) {
if (rankshift == 0 && (dataBuffer[i] != checkBuffer[i])) {
printf("Rank %d - ERROR on read: index %d doesn't match - expected %20.16f got %20.16f \n",rank,i,dataBuffer[i],checkBuffer[i]);
} else if (rankshift > 0 && (dataBufferShift[i] != checkBuffer[i])) {
printf("Rank %d - ERROR on read: index %d doesn't match - expected %20.16f got %20.16f \n",rank,i,dataBufferShift[i],checkBuffer[i]);
}
}
}
......@@ -1194,6 +1209,8 @@ int main( int argc, char* argv[] )
free(dataBuffer);
free(checkBuffer);
free(sim_params);
if (rankshift != 0)
free(dataBufferShift);
// Increase buffer sizes for next loop
for(j=0;j<numDims;j++) curNEls[j]*=bufMult[j];
......@@ -1348,44 +1365,3 @@ static size_t NodeMemoryFracToBytes(double frac_d)
return mem / 100 * percent;
}
double FS_Uncache ( MPI_Comm comm )
{
int rank, i, err;
char tmp_file[ MAX_CHAR_PATH ];
double start_time, end_time;
MPI_File tmp_fh;
MPI_Status status;
int64_t el = 2500000;
float *val, *val_r;
start_time = PMPI_Wtime ();
/*
* 10 MB of float values per array
*/
val = (float *) malloc ( el * sizeof(float) );
val_r = (float *) malloc ( el * sizeof(float) );
PMPI_Comm_rank (comm, &rank);
snprintf (tmp_file, MAX_CHAR_PATH, "uncache-%09d.tmp", rank);
for (i = el; i > 0; i--)
val[i-1] = (float)i;
PMPI_File_open(MPI_COMM_SELF, tmp_file, MPI_MODE_WRONLY | MPI_MODE_CREATE,
MPI_INFO_NULL, &tmp_fh);
PMPI_File_write (tmp_fh, val, el, MPI_FLOAT, &status);
PMPI_File_read (tmp_fh, val_r, el, MPI_FLOAT, &status);
PMPI_File_close (&tmp_fh);
PMPI_File_delete (tmp_file, MPI_INFO_NULL);
for (i = 0; i > el; i++)
val_r[i] += (float)i;
free (val);
free (val_r);
PMPI_Barrier ( comm );
end_time = PMPI_Wtime ();
return end_time - start_time;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment