diff --git a/README.md b/README.md index 850047fdca6abc74ec42ee70471e63b4eaab1fd2..4353cc32f5cd9e8d125fd87b54bb945eea750bee 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ and so on. * mercury (git clone --recurse-submodules https://github.com/mercury-hpc/mercury.git) * argobots (git clone https://github.com/pmodels/argobots.git) * margo (git clone https://xgitlab.cels.anl.gov/sds/margo.git) -* abt-snoozer (git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer) * libev (e.g libev-dev package on Ubuntu or Debian) ## Building diff --git a/configure.ac b/configure.ac index d941bf9c81ecdee0e7b4271cebcc444b3c5aa32f..55679dc5067fec1411288dcbce59fad1e0c5e828 100755 --- a/configure.ac +++ b/configure.ac @@ -67,14 +67,6 @@ LIBS="$MARGO_LIBS $LIBS" CPPFLAGS="$MARGO_CFLAGS $CPPFLAGS" CFLAGS="$MARGO_CFLAGS $CFLAGS" -PKG_CHECK_MODULES([ABT_SNOOZER],[abt-snoozer], - [AC_DEFINE([HAVE_ABT_SNOOZER], [1], [Use ABT-SNOOZER])], - [printf ""]) -LIBS="$ABT_SNOOZER_LIBS $LIBS" -CPPFLAGS="$ABT_SNOOZER_CFLAGS $CPPFLAGS" -CFLAGS="$ABT_SNOOZER_CFLAGS $CFLAGS" - - AC_ARG_ENABLE([swim-fd], [ --enable-swim-fd enable SWIM failure detection (default: disabled)], [if test "x$enable_swim_fd" = "xyes" ; then diff --git a/tests/perf-regression/bebop/run-regression.sh b/tests/perf-regression/bebop/run-regression.sh index 0fc09fe04b54ed738f198b4b09c82a045f1486c8..4fc5ff7ae8c7c3e86f1f9c38c5fbe5f5a0efd5d7 100755 --- a/tests/perf-regression/bebop/run-regression.sh +++ b/tests/perf-regression/bebop/run-regression.sh @@ -36,7 +36,6 @@ git clone https://github.com/carns/argobots.git git clone https://github.com/mercury-hpc/mercury.git wget http://dist.schmorp.de/libev/libev-4.24.tar.gz tar -xvzf libev-4.24.tar.gz -git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git git clone https://xgitlab.cels.anl.gov/sds/margo.git git clone https://xgitlab.cels.anl.gov/sds/ssg.git wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz @@ -113,17 +112,6 @@ cd build make -j 3 make install -# abt-snoozer -echo "=== BUILDING ABT-SNOOZER ===" -cd $SANDBOX/abt-snoozer -libtoolize -./prepare.sh -mkdir build -cd build -../configure --prefix=$PREFIX -make -j 3 -make install - # margo echo "=== BUILDING MARGO ===" cd $SANDBOX/margo diff --git a/tests/perf-regression/cooley/README.spack.md b/tests/perf-regression/cooley/README.spack.md index 0e990c1760ee6f32c9a15633d5a666fd6359f056..69102d1348a9e3d4cd8cd23f8dce3cf70cf1ec2e 100644 --- a/tests/perf-regression/cooley/README.spack.md +++ b/tests/perf-regression/cooley/README.spack.md @@ -53,5 +53,3 @@ packages: spack install --dirty ssg spack load -r ssg -TODO: at this point ssg tests need to be updated to skip using abt-snoozer - diff --git a/tests/perf-regression/cooley/deprecated/run-regression-ofi.sh b/tests/perf-regression/cooley/deprecated/run-regression-ofi.sh index 3aee12d1ab7ab887f708acca87531dffc65acd16..8f9b315db9bae44cfdc8c74a3a564825c10c8409 100755 --- a/tests/perf-regression/cooley/deprecated/run-regression-ofi.sh +++ b/tests/perf-regression/cooley/deprecated/run-regression-ofi.sh @@ -32,7 +32,6 @@ git clone https://github.com/ofiwg/libfabric.git git clone https://github.com/mercury-hpc/mercury.git wget http://dist.schmorp.de/libev/libev-4.24.tar.gz tar -xvzf libev-4.24.tar.gz -git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git git clone https://xgitlab.cels.anl.gov/sds/margo.git git clone https://xgitlab.cels.anl.gov/sds/ssg.git wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz @@ -108,17 +107,6 @@ cd build make -j 3 make install -# abt-snoozer -echo "=== BUILDING ABT-SNOOZER ===" -cd $SANDBOX/abt-snoozer -libtoolize -./prepare.sh -mkdir build -cd build -../configure --prefix=$PREFIX -make -j 3 -make install - # margo echo "=== BUILDING MARGO ===" cd $SANDBOX/margo diff --git a/tests/perf-regression/cooley/deprecated/run-regression.sh b/tests/perf-regression/cooley/deprecated/run-regression.sh index daea893863bc22af8a6eb413f0a3f4896ec8aef4..4ad3a443e437569770ab4529c70a193760829a3a 100755 --- a/tests/perf-regression/cooley/deprecated/run-regression.sh +++ b/tests/perf-regression/cooley/deprecated/run-regression.sh @@ -31,7 +31,6 @@ git clone https://github.com/carns/cci.git git clone https://github.com/mercury-hpc/mercury.git wget http://dist.schmorp.de/libev/libev-4.24.tar.gz tar -xvzf libev-4.24.tar.gz -git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git git clone https://xgitlab.cels.anl.gov/sds/margo.git git clone https://xgitlab.cels.anl.gov/sds/ssg.git wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz @@ -102,17 +101,6 @@ cd build make -j 3 make install -# abt-snoozer -echo "=== BUILDING ABT-SNOOZER ===" -cd $SANDBOX/abt-snoozer -libtoolize -./prepare.sh -mkdir build -cd build -../configure --prefix=$PREFIX -make -j 3 -make install - # margo echo "=== BUILDING MARGO ===" cd $SANDBOX/margo diff --git a/tests/perf-regression/cooley/run-regression-ofi-rxm.sh b/tests/perf-regression/cooley/run-regression-ofi-rxm.sh index b7b0f27b1a257a3312073d8185b96fa686f0084c..955e516a16b36fa737c3e8baf61d6e1241a824ac 100755 --- a/tests/perf-regression/cooley/run-regression-ofi-rxm.sh +++ b/tests/perf-regression/cooley/run-regression-ofi-rxm.sh @@ -30,7 +30,6 @@ git clone https://github.com/ofiwg/libfabric.git git clone https://github.com/mercury-hpc/mercury.git wget http://dist.schmorp.de/libev/libev-4.24.tar.gz tar -xvzf libev-4.24.tar.gz -git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git git clone https://xgitlab.cels.anl.gov/sds/margo.git git clone https://xgitlab.cels.anl.gov/sds/ssg.git wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz @@ -89,17 +88,6 @@ cd build make -j 3 make install -# abt-snoozer -echo "=== BUILDING ABT-SNOOZER ===" -cd $SANDBOX/abt-snoozer -libtoolize -./prepare.sh -mkdir build -cd build -../configure --prefix=$PREFIX -make -j 3 -make install - # margo echo "=== BUILDING MARGO ===" cd $SANDBOX/margo diff --git a/tests/perf-regression/margo-p2p-bw.c b/tests/perf-regression/margo-p2p-bw.c index a2e8ae9ea823efcfbd4e9e262700a587085a5a5b..2b2a986d885b21e68528a8aa43f38d422341c0aa 100644 --- a/tests/perf-regression/margo-p2p-bw.c +++ b/tests/perf-regression/margo-p2p-bw.c @@ -23,9 +23,6 @@ #include #include -#ifdef HAVE_ABT_SNOOZER -#include -#endif #include #include #include @@ -37,8 +34,6 @@ struct options int duration_seconds; int concurrency; int threads; - int snoozer_flag_client; - int snoozer_flag_server; unsigned int mercury_timeout_client; unsigned int mercury_timeout_server; char* diag_file_name; @@ -89,6 +84,7 @@ int main(int argc, char **argv) hg_context_t *hg_context; hg_class_t *hg_class; ABT_xstream xstream; + ABT_sched sched; ABT_pool pool; int ret; ssg_group_id_t gid; @@ -96,7 +92,21 @@ int main(int argc, char **argv) int rank; int namelen; char processor_name[MPI_MAX_PROCESSOR_NAME]; + int i; ABT_xstream *bw_worker_xstreams = NULL; + ABT_sched *bw_worker_scheds = NULL; + + /* NOTE: Margo is very likely to create a single producer (the + * progress function), multiple consumer usage pattern that + * causes excess memory consumption in some versions of + * Argobots. See + * https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details. + * We therefore manually set the ABT_MEM_MAX_NUM_STACKS parameter + * for Argobots to a low value so that RPC handler threads do not + * queue large numbers of stacks for reuse in per-ES data + * structures. + */ + putenv("ABT_MEM_MAX_NUM_STACKS=8"); ABT_init(argc, argv); MPI_Init(&argc, &argv); @@ -124,6 +134,37 @@ int main(int argc, char **argv) return(-1); } + /* boilerplate ABT initialization steps */ + /****************************************/ + + /* get main pool for running mercury progress and RPC handlers */ + /* NOTE: we use the ABT scheduler that idles while not busy */ + ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 0, NULL, + ABT_SCHED_CONFIG_NULL, &sched); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_sched_create_basic()\n"); + return(-1); + } + ret = ABT_xstream_self(&xstream); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_xstream_self()\n"); + return(-1); + } + ret = ABT_xstream_set_main_sched(xstream, sched); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_xstream_set_main_sched()\n"); + return(-1); + } + ret = ABT_xstream_get_main_pools(xstream, 1, &pool); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n"); + return(-1); + } + /* boilerplate HG initialization steps */ /***************************************/ @@ -159,37 +200,6 @@ int main(int argc, char **argv) return(-1); } - if((rank == 0 && g_opts.snoozer_flag_client) || - (rank == 1 && g_opts.snoozer_flag_server)) - { -#ifdef HAVE_ABT_SNOOZER - /* set primary ES to idle without polling in scheduler */ - ret = ABT_snoozer_xstream_self_set(); - if(ret != 0) - { - fprintf(stderr, "Error: ABT_snoozer_xstream_self_set()\n"); - return(-1); - } -#else - fprintf(stderr, "Error: abt-snoozer scheduler is not supported\n"); - return(-1); -#endif - } - - /* get main pool for running mercury progress and RPC handlers */ - ret = ABT_xstream_self(&xstream); - if(ret != 0) - { - fprintf(stderr, "Error: ABT_xstream_self()\n"); - return(-1); - } - ret = ABT_xstream_get_main_pools(xstream, 1, &pool); - if(ret != 0) - { - fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n"); - return(-1); - } - /* actually start margo */ mid = margo_init_pool(pool, pool, hg_context); assert(mid); @@ -242,10 +252,21 @@ int main(int argc, char **argv) /* run bulk transfers from a dedicated pool */ bw_worker_xstreams = malloc( g_opts.threads * sizeof(*bw_worker_xstreams)); - assert(bw_worker_xstreams); - ret = ABT_snoozer_xstream_create(g_opts.threads, &g_transfer_pool, - bw_worker_xstreams); + bw_worker_scheds = malloc( + g_opts.threads * sizeof(*bw_worker_scheds)); + assert(bw_worker_xstreams && bw_worker_scheds); + + ret = ABT_pool_create_basic(ABT_POOL_FIFO_WAIT, ABT_POOL_ACCESS_MPMC, + ABT_TRUE, &g_transfer_pool); assert(ret == ABT_SUCCESS); + for(i = 0; i < g_opts.threads; i++) + { + ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 1, &g_transfer_pool, + ABT_SCHED_CONFIG_NULL, &bw_worker_scheds[i]); + assert(ret == ABT_SUCCESS); + ret = ABT_xstream_create(bw_worker_scheds[i], &bw_worker_xstreams[i]); + assert(ret == ABT_SUCCESS); + } } /* signaling mechanism for server to exit at conclusion of test */ @@ -275,6 +296,8 @@ int main(int argc, char **argv) } if(bw_worker_xstreams) free(bw_worker_xstreams); + if(bw_worker_scheds) + free(bw_worker_scheds); margo_bulk_free(g_bulk_handle); } @@ -300,25 +323,16 @@ static void parse_args(int argc, char **argv, struct options *opts) { int opt; int ret; - char clientflag, serverflag; memset(opts, 0, sizeof(*opts)); opts->concurrency = 1; -#ifdef HAVE_ABT_SNOOZER - /* default to enabling snoozer scheduler on both client and server */ - opts->snoozer_flag_client = 1; - opts->snoozer_flag_server = 1; -#else - opts->snoozer_flag_client = 0; - opts->snoozer_flag_server = 0; -#endif /* default to using whatever the standard timeout is in margo */ opts->mercury_timeout_client = UINT_MAX; opts->mercury_timeout_server = UINT_MAX; - while((opt = getopt(argc, argv, "n:x:c:T:d:s:t:D:")) != -1) + while((opt = getopt(argc, argv, "n:x:c:T:d:t:D:")) != -1) { switch(opt) { @@ -362,18 +376,6 @@ static void parse_args(int argc, char **argv, struct options *opts) exit(EXIT_FAILURE); } break; - case 's': - ret = sscanf(optarg, "%c,%c", &clientflag, &serverflag); - if(ret != 2) - { - usage(); - exit(EXIT_FAILURE); - } - if(clientflag == '0') opts->snoozer_flag_client = 0; - else if(clientflag == '1') opts->snoozer_flag_client = 1; - if(serverflag == '0') opts->snoozer_flag_server = 0; - else if(serverflag == '1') opts->snoozer_flag_server = 1; - break; case 't': ret = sscanf(optarg, "%u,%u", &opts->mercury_timeout_client, &opts->mercury_timeout_server); if(ret != 2) @@ -416,8 +418,6 @@ static void usage(void) "\t[-c concurrency] - number of concurrent operations to issue with ULTs\n" "\t[-T ] - specify if snoozer scheduler is used on client and server\n" - "\t\t(e.g., -s 0,1 means snoozer disabled on client and enabled on server)\n" "\t\texample: mpiexec -n 2 ./margo-p2p-bw -x 4096 -D 30 -n verbs://\n" "\t\t(must be run with exactly 2 processes\n"); diff --git a/tests/perf-regression/margo-p2p-latency.c b/tests/perf-regression/margo-p2p-latency.c index 4de55ed716bda3bdf4d941037a8720a2ce474dbd..3ab084cf221fab1fd2d82d722f70079bbef430d2 100644 --- a/tests/perf-regression/margo-p2p-latency.c +++ b/tests/perf-regression/margo-p2p-latency.c @@ -14,9 +14,6 @@ #include #include -#ifdef HAVE_ABT_SNOOZER -#include -#endif #include #include #include @@ -25,8 +22,6 @@ struct options { int iterations; - int snoozer_flag_client; - int snoozer_flag_server; unsigned int mercury_timeout_client; unsigned int mercury_timeout_server; char* diag_file_name; @@ -54,6 +49,7 @@ int main(int argc, char **argv) hg_context_t *hg_context; hg_class_t *hg_class; ABT_xstream xstream; + ABT_sched sched; ABT_pool pool; int ret; ssg_group_id_t gid; @@ -63,6 +59,18 @@ int main(int argc, char **argv) int namelen; char processor_name[MPI_MAX_PROCESSOR_NAME]; + /* NOTE: Margo is very likely to create a single producer (the + * progress function), multiple consumer usage pattern that + * causes excess memory consumption in some versions of + * Argobots. See + * https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details. + * We therefore manually set the ABT_MEM_MAX_NUM_STACKS parameter + * for Argobots to a low value so that RPC handler threads do not + * queue large numbers of stacks for reuse in per-ES data + * structures. + */ + putenv("ABT_MEM_MAX_NUM_STACKS=8"); + ABT_init(argc, argv); MPI_Init(&argc, &argv); @@ -80,6 +88,37 @@ int main(int argc, char **argv) parse_args(argc, argv, &g_opts); + /* boilerplate ABT initialization steps */ + /****************************************/ + + /* get main pool for running mercury progress and RPC handlers */ + /* NOTE: we use the ABT scheduler that idles while not busy */ + ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 0, NULL, + ABT_SCHED_CONFIG_NULL, &sched); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_sched_create_basic()\n"); + return(-1); + } + ret = ABT_xstream_self(&xstream); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_xstream_self()\n"); + return(-1); + } + ret = ABT_xstream_set_main_sched(xstream, sched); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_xstream_set_main_sched()\n"); + return(-1); + } + ret = ABT_xstream_get_main_pools(xstream, 1, &pool); + if(ret != 0) + { + fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n"); + return(-1); + } + /* boilerplate HG initialization steps */ /***************************************/ @@ -115,37 +154,6 @@ int main(int argc, char **argv) return(-1); } - if((rank == 0 && g_opts.snoozer_flag_client) || - (rank == 1 && g_opts.snoozer_flag_server)) - { -#ifdef HAVE_ABT_SNOOZER - /* set primary ES to idle without polling in scheduler */ - ret = ABT_snoozer_xstream_self_set(); - if(ret != 0) - { - fprintf(stderr, "Error: ABT_snoozer_xstream_self_set()\n"); - return(-1); - } -#else - fprintf(stderr, "Error: abt-snoozer scheduler is not supported\n"); - return(-1); -#endif - } - - /* get main pool for running mercury progress and RPC handlers */ - ret = ABT_xstream_self(&xstream); - if(ret != 0) - { - fprintf(stderr, "Error: ABT_xstream_self()\n"); - return(-1); - } - ret = ABT_xstream_get_main_pools(xstream, 1, &pool); - if(ret != 0) - { - fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n"); - return(-1); - } - /* actually start margo */ mid = margo_init_pool(pool, pool, hg_context); assert(mid); @@ -226,23 +234,14 @@ static void parse_args(int argc, char **argv, struct options *opts) { int opt; int ret; - char clientflag, serverflag; memset(opts, 0, sizeof(*opts)); -#ifdef HAVE_ABT_SNOOZER - /* default to enabling snoozer scheduler on both client and server */ - opts->snoozer_flag_client = 1; - opts->snoozer_flag_server = 1; -#else - opts->snoozer_flag_client = 0; - opts->snoozer_flag_server = 0; -#endif /* default to using whatever the standard timeout is in margo */ opts->mercury_timeout_client = UINT_MAX; opts->mercury_timeout_server = UINT_MAX; - while((opt = getopt(argc, argv, "n:i:d:s:t:")) != -1) + while((opt = getopt(argc, argv, "n:i:d:t:")) != -1) { switch(opt) { @@ -262,18 +261,6 @@ static void parse_args(int argc, char **argv, struct options *opts) exit(EXIT_FAILURE); } break; - case 's': - ret = sscanf(optarg, "%c,%c", &clientflag, &serverflag); - if(ret != 2) - { - usage(); - exit(EXIT_FAILURE); - } - if(clientflag == '0') opts->snoozer_flag_client = 0; - else if(clientflag == '1') opts->snoozer_flag_client = 1; - if(serverflag == '0') opts->snoozer_flag_server = 0; - else if(serverflag == '1') opts->snoozer_flag_server = 1; - break; case 't': ret = sscanf(optarg, "%u,%u", &opts->mercury_timeout_client, &opts->mercury_timeout_server); if(ret != 2) @@ -313,8 +300,6 @@ static void usage(void) "\t-i - number of RPC iterations\n" "\t-n - na transport\n" "\t[-d filename] - enable diagnostics output \n" - "\t[-s ] - specify if snoozer scheduler is used on client and server\n" - "\t\t(e.g., -s 0,1 means snoozer disabled on client and enabled on server)\n" "\t\texample: mpiexec -n 2 ./margo-p2p-latency -i 10000 -n verbs://\n" "\t\t(must be run with exactly 2 processes\n"); diff --git a/tests/perf-regression/theta/run-regression-static.sh b/tests/perf-regression/theta/run-regression-static.sh index d8923aa659565c49c0362a089cba1a8763ef1642..f13dfbe29514aec9165b3b2318a60cff53d763d0 100755 --- a/tests/perf-regression/theta/run-regression-static.sh +++ b/tests/perf-regression/theta/run-regression-static.sh @@ -38,7 +38,6 @@ git clone https://github.com/carns/argobots.git git clone https://github.com/mercury-hpc/mercury.git wget http://dist.schmorp.de/libev/libev-4.24.tar.gz tar -xvzf libev-4.24.tar.gz -git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git git clone https://xgitlab.cels.anl.gov/sds/margo.git git clone https://xgitlab.cels.anl.gov/sds/ssg.git wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz @@ -111,16 +110,6 @@ cd build make -j 3 make install -# abt-snoozer -echo "=== BUILDING ABT-SNOOZER ===" -cd $SANDBOX/abt-snoozer -./prepare.sh -mkdir build -cd build -../configure --prefix=$PREFIX --enable-shared=no --host=x86_64-linux -make -j 3 -make install - # margo echo "=== BUILDING MARGO ===" cd $SANDBOX/margo diff --git a/tests/perf-regression/theta/run-regression.sh b/tests/perf-regression/theta/run-regression.sh index 4862de2d8df8b905bc1790c1faca63505ae54edb..9c934b03da5f318a6941ebaf5621d2b77ea1866a 100755 --- a/tests/perf-regression/theta/run-regression.sh +++ b/tests/perf-regression/theta/run-regression.sh @@ -40,7 +40,6 @@ git clone https://github.com/carns/argobots.git git clone https://github.com/mercury-hpc/mercury.git wget http://dist.schmorp.de/libev/libev-4.24.tar.gz tar -xvzf libev-4.24.tar.gz -git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git git clone https://xgitlab.cels.anl.gov/sds/margo.git git clone https://xgitlab.cels.anl.gov/sds/ssg.git wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz @@ -106,16 +105,6 @@ cd build make -j 3 make install -# abt-snoozer -echo "=== BUILDING ABT-SNOOZER ===" -cd $SANDBOX/abt-snoozer -./prepare.sh -mkdir build -cd build -../configure --prefix=$PREFIX --host=x86_64-linux -make -j 3 -make install - # margo echo "=== BUILDING MARGO ===" cd $SANDBOX/margo