Commit 029a9f0d authored by Richard Zamora's avatar Richard Zamora
Browse files

topology adjustments for mira/theta testing

parent 3fad11b2
......@@ -418,6 +418,7 @@ int main( int argc, char* argv[] )
int64_t* offset_size = (int64_t*) malloc(sizeof(int64_t*) * (num_chunks) );
// Real expected data pattern for first data size (Dim-1 Only)
// (Set data size to ensure ranks will favor certain aggs)
if ( div_data_I && !(strcmp(div_data_I,"0") == 0) && (numDims == 1)) {
/*
......@@ -441,6 +442,16 @@ int main( int argc, char* argv[] )
offset_size[i] = (int64_t) (i * dperiod + rank * chunk_size);
}
// Real expected data pattern for first data size (Dim-1 Only)
} else if (numDims == 1) {
chunk_size = minNEls[0] * sizeof( double );
dperiod = (int64_t) (chunk_size * nprocs);
for (i=0; i<num_chunks; i++) {
data_size[i] = (int64_t) (chunk_size);
offset_size[i] = (int64_t) (i * dperiod + rank * chunk_size);
}
// Create a fake data set that will have all ranks writing to all aggs
} else {
......@@ -453,7 +464,7 @@ int main( int argc, char* argv[] )
// Define agg_array
char agg_array[256];
get_cb_config_list ( data_size, offset_size, num_chunks, &agg_array[0], agg_size, MPI_COMM_WORLD, agg_count );
get_cb_config_list ( data_size, offset_size, num_chunks, &agg_array[0], agg_size, agg_count, MPI_COMM_WORLD );
//if (rank==0) printf("Aggregator List:\n%s\n", agg_array);
MPI_Barrier(comm);
free(data_size);
......
......@@ -40,7 +40,7 @@
* from the MPI_STATUS and then exit the program. This macro is borrowed
* directly from the HPC-IOR code
*/
#define MPI_CHECK_H5(MPI_STATUS, MSG) do { \
#define MPI_CHECK_H5(MPI_STATUS, MSG) do { \
char resultString[MPI_MAX_ERROR_STRING]; \
int resultLength; \
\
......@@ -359,72 +359,82 @@ int distance_to_io_node ( int src_rank ) {
return 1;
}
/*-------------------------------------------------------------------------
* Function: get_cb_props
*
* Purpose: Returns aggregator settings for current FS.
* cb_override allows calling program to set the cb_node value.
*
* Return: Void
*
*-------------------------------------------------------------------------
*/
void get_cb_props( int64_t *buffer_size, int64_t *nb_aggr, char* fname, char* cb_override ) {
int rank, nprocs;
int info_flag;
MPI_File fh;
MPI_Info mpi_file_info;
char info_value[MPI_MAX_INFO_VAL];
int mpi_code, striping_unit, striping_factor, co_ratio;
int rank, nprocs;
int info_flag;
MPI_File fh;
MPI_Info mpi_file_info;
char info_value[MPI_MAX_INFO_VAL];
int mpi_code, striping_unit, striping_factor, co_ratio;
MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
MPI_Comm_size ( MPI_COMM_WORLD, &nprocs );
MPI_Comm_rank ( MPI_COMM_WORLD, &rank );
MPI_Comm_size ( MPI_COMM_WORLD, &nprocs );
mpi_code = MPI_File_open(MPI_COMM_WORLD, fname, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
mpi_code = MPI_File_open(MPI_COMM_WORLD, fname, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
#ifdef THETA
MPI_File_get_info(fh, &mpi_file_info);
mpi_code = MPI_Info_get(mpi_file_info, "romio_filesystem_type", MPI_MAX_INFO_VAL, info_value, &info_flag);
if (rank == 0) printf("romio_filesystem_type is :%s:\n",info_value);
MPI_File_get_info(fh, &mpi_file_info);
mpi_code = MPI_Info_get(mpi_file_info, "romio_filesystem_type", MPI_MAX_INFO_VAL, info_value, &info_flag);
if (rank == 0) printf("romio_filesystem_type is :%s:\n",info_value);
// striping_unit is the size of the stripe (in bytes)
mpi_code = MPI_Info_get(mpi_file_info, "striping_unit", MPI_MAX_INFO_VAL, info_value, &info_flag);
striping_unit = atoi(info_value);
if (rank == 0) printf("striping_unit is :%s:\n",info_value);
// striping_unit is the size of the stripe (in bytes)
mpi_code = MPI_Info_get(mpi_file_info, "striping_unit", MPI_MAX_INFO_VAL, info_value, &info_flag);
striping_unit = atoi(info_value);
if (rank == 0) printf("striping_unit is :%s:\n",info_value);
// striping_factor is the number of stripes used for the file
mpi_code = MPI_Info_get(mpi_file_info, "striping_factor", MPI_MAX_INFO_VAL, info_value, &info_flag);
striping_factor = atoi(info_value);
if (rank == 0) printf("striping_factor is :%s:\n",info_value);
// striping_factor is the number of stripes used for the file
mpi_code = MPI_Info_get(mpi_file_info, "striping_factor", MPI_MAX_INFO_VAL, info_value, &info_flag);
striping_factor = atoi(info_value);
if (rank == 0) printf("striping_factor is :%s:\n",info_value);
// Number of aggregators = "striping_factor" X "romio_lustre_co_ratio"
mpi_code = MPI_Info_get(mpi_file_info, "romio_lustre_co_ratio", MPI_MAX_INFO_VAL, info_value, &info_flag);
co_ratio = atoi(info_value);
if (rank == 0) printf("romio_lustre_co_ratio is :%s:\n",info_value);
// Number of aggregators = "striping_factor" X "romio_lustre_co_ratio"
mpi_code = MPI_Info_get(mpi_file_info, "romio_lustre_co_ratio", MPI_MAX_INFO_VAL, info_value, &info_flag);
co_ratio = atoi(info_value);
if (rank == 0) printf("romio_lustre_co_ratio is :%s:\n",info_value);
*buffer_size = (int64_t) striping_unit;
*nb_aggr = (int64_t) (striping_factor * co_ratio);
*buffer_size = (int64_t) striping_unit;
*nb_aggr = (int64_t) (striping_factor * co_ratio);
#elif ROMIO
#elif (ROMIO || BGQ)
MPI_File_get_info(fh, &mpi_file_info);
MPI_File_get_info(fh, &mpi_file_info);
/* Override the number of aggeregators (if HDF5_CB_NODES_OVERRIDE env is set) */
if ( cb_override && !(strcmp(cb_override,"0") == 0) ) {
MPI_Info_set(mpi_file_info, "cb_nodes", cb_override);
}
/* Override the number of aggeregators (if HDF5_CB_NODES_OVERRIDE env is set) */
if ( cb_override && !(strcmp(cb_override,"0") == 0) ) {
MPI_Info_set(mpi_file_info, "cb_nodes", cb_override);
}
mpi_code = MPI_Info_get(mpi_file_info, "cb_nodes", MPI_MAX_INFO_VAL, info_value, &info_flag);
*nb_aggr = (int64_t) atoi(info_value);
if (rank == 0) printf("cb_nodes is :%lld:\n",*nb_aggr);
mpi_code = MPI_Info_get(mpi_file_info, "cb_nodes", MPI_MAX_INFO_VAL, info_value, &info_flag);
*nb_aggr = (int64_t) atoi(info_value);
if (rank == 0) printf("cb_nodes is :%lld:\n",*nb_aggr);
mpi_code = MPI_Info_get(mpi_file_info, "cb_buffer_size", MPI_MAX_INFO_VAL, info_value, &info_flag);
*buffer_size = (int64_t) atoi(info_value);
if (rank == 0) printf("cb_buffer_size is :%lld:\n",*buffer_size);
mpi_code = MPI_Info_get(mpi_file_info, "cb_buffer_size", MPI_MAX_INFO_VAL, info_value, &info_flag);
*buffer_size = (int64_t) atoi(info_value);
if (rank == 0) printf("cb_buffer_size is :%lld:\n",*buffer_size);
#else
*nb_aggr = 1;
*buffer_size = 65536;
*nb_aggr = 1;
*buffer_size = 65536;
#endif
MPI_File_close( &fh );
if (rank == 0) printf("Using %lld aggregators with %lld buffer size\n",*nb_aggr,*buffer_size);
MPI_File_close( &fh );
if (rank == 0) printf("Using %lld aggregators with %lld buffer size\n",*nb_aggr,*buffer_size);
return;
return;
}
......@@ -577,7 +587,7 @@ int add_chunk ( int64_t datalen, int64_t offset, int64_t buffer_size, int64_t nb
*
*-------------------------------------------------------------------------
*/
int get_cb_config_list ( int64_t* data_lens, int64_t* offsets, int data_len, char* hint_str, int64_t buffer_size, MPI_Comm comm, int64_t nb_aggr )
int get_cb_config_list ( int64_t* data_lens, int64_t* offsets, int data_len, char* hint_str, int64_t buffer_size, int64_t nb_aggr, MPI_Comm comm )
{
int rank, nprocs, i, r, resultlen;
int* agg_list;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment