remove abt-snoozer dependency from tests/configure

README.md

@@ -45,7 +45,6 @@ and so on.
 * mercury (git clone --recurse-submodules https://github.com/mercury-hpc/mercury.git)
 * argobots (git clone https://github.com/pmodels/argobots.git)
 * margo (git clone https://xgitlab.cels.anl.gov/sds/margo.git)
-* abt-snoozer (git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer)
 * libev (e.g libev-dev package on Ubuntu or Debian)
 
 ## Building

configure.ac

@@ -67,14 +67,6 @@ LIBS="$MARGO_LIBS $LIBS"
 CPPFLAGS="$MARGO_CFLAGS $CPPFLAGS"
 CFLAGS="$MARGO_CFLAGS $CFLAGS"
 
-PKG_CHECK_MODULES([ABT_SNOOZER],[abt-snoozer],
-    [AC_DEFINE([HAVE_ABT_SNOOZER], [1], [Use ABT-SNOOZER])],
-    [printf ""])
-LIBS="$ABT_SNOOZER_LIBS $LIBS"
-CPPFLAGS="$ABT_SNOOZER_CFLAGS $CPPFLAGS"
-CFLAGS="$ABT_SNOOZER_CFLAGS $CFLAGS"
-
-
 AC_ARG_ENABLE([swim-fd],
     [  --enable-swim-fd        enable SWIM failure detection (default: disabled)],
     [if test "x$enable_swim_fd" = "xyes" ; then

tests/perf-regression/bebop/run-regression.sh

@@ -36,7 +36,6 @@ git clone https://github.com/carns/argobots.git
 git clone https://github.com/mercury-hpc/mercury.git
 wget http://dist.schmorp.de/libev/libev-4.24.tar.gz
 tar -xvzf libev-4.24.tar.gz
-git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git
 git clone https://xgitlab.cels.anl.gov/sds/margo.git
 git clone https://xgitlab.cels.anl.gov/sds/ssg.git
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
@@ -113,17 +112,6 @@ cd build
 make -j 3
 make install
 
-# abt-snoozer
-echo "=== BUILDING ABT-SNOOZER ==="
-cd $SANDBOX/abt-snoozer
-libtoolize
-./prepare.sh
-mkdir build
-cd build
-../configure --prefix=$PREFIX 
-make -j 3
-make install
-
 # margo
 echo "=== BUILDING MARGO ==="
 cd $SANDBOX/margo

tests/perf-regression/cooley/README.spack.md

@@ -53,5 +53,3 @@ packages:
 spack install --dirty ssg
 spack load -r ssg
 
-TODO: at this point ssg tests need to be updated to skip using abt-snoozer
-

tests/perf-regression/cooley/deprecated/run-regression-ofi.sh

@@ -32,7 +32,6 @@ git clone https://github.com/ofiwg/libfabric.git
 git clone https://github.com/mercury-hpc/mercury.git
 wget http://dist.schmorp.de/libev/libev-4.24.tar.gz
 tar -xvzf libev-4.24.tar.gz
-git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git
 git clone https://xgitlab.cels.anl.gov/sds/margo.git
 git clone https://xgitlab.cels.anl.gov/sds/ssg.git
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
@@ -108,17 +107,6 @@ cd build
 make -j 3
 make install
 
-# abt-snoozer
-echo "=== BUILDING ABT-SNOOZER ==="
-cd $SANDBOX/abt-snoozer
-libtoolize
-./prepare.sh
-mkdir build
-cd build
-../configure --prefix=$PREFIX 
-make -j 3
-make install
-
 # margo
 echo "=== BUILDING MARGO ==="
 cd $SANDBOX/margo

tests/perf-regression/cooley/deprecated/run-regression.sh

@@ -31,7 +31,6 @@ git clone https://github.com/carns/cci.git
 git clone https://github.com/mercury-hpc/mercury.git
 wget http://dist.schmorp.de/libev/libev-4.24.tar.gz
 tar -xvzf libev-4.24.tar.gz
-git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git
 git clone https://xgitlab.cels.anl.gov/sds/margo.git
 git clone https://xgitlab.cels.anl.gov/sds/ssg.git
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
@@ -102,17 +101,6 @@ cd build
 make -j 3
 make install
 
-# abt-snoozer
-echo "=== BUILDING ABT-SNOOZER ==="
-cd $SANDBOX/abt-snoozer
-libtoolize
-./prepare.sh
-mkdir build
-cd build
-../configure --prefix=$PREFIX 
-make -j 3
-make install
-
 # margo
 echo "=== BUILDING MARGO ==="
 cd $SANDBOX/margo

tests/perf-regression/cooley/run-regression-ofi-rxm.sh

@@ -30,7 +30,6 @@ git clone https://github.com/ofiwg/libfabric.git
 git clone https://github.com/mercury-hpc/mercury.git
 wget http://dist.schmorp.de/libev/libev-4.24.tar.gz
 tar -xvzf libev-4.24.tar.gz
-git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git
 git clone https://xgitlab.cels.anl.gov/sds/margo.git
 git clone https://xgitlab.cels.anl.gov/sds/ssg.git
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
@@ -89,17 +88,6 @@ cd build
 make -j 3
 make install
 
-# abt-snoozer
-echo "=== BUILDING ABT-SNOOZER ==="
-cd $SANDBOX/abt-snoozer
-libtoolize
-./prepare.sh
-mkdir build
-cd build
-../configure --prefix=$PREFIX 
-make -j 3
-make install
-
 # margo
 echo "=== BUILDING MARGO ==="
 cd $SANDBOX/margo

tests/perf-regression/margo-p2p-bw.c

@@ -23,9 +23,6 @@
 #include <mpi.h>
 
 #include <margo.h>
-#ifdef HAVE_ABT_SNOOZER
-#include <abt-snoozer.h>
-#endif
 #include <mercury.h>
 #include <abt.h>
 #include <ssg.h>
@@ -37,8 +34,6 @@ struct options
     int duration_seconds;
     int concurrency;
     int threads;
-    int snoozer_flag_client;
-    int snoozer_flag_server;
     unsigned int mercury_timeout_client;
     unsigned int mercury_timeout_server;
     char* diag_file_name;
@@ -89,6 +84,7 @@ int main(int argc, char **argv)
     hg_context_t *hg_context;
     hg_class_t *hg_class;
     ABT_xstream xstream;
+    ABT_sched sched;
     ABT_pool pool;
     int ret;
     ssg_group_id_t gid;
@@ -96,7 +92,21 @@ int main(int argc, char **argv)
     int rank;
     int namelen;
     char processor_name[MPI_MAX_PROCESSOR_NAME];
+    int i;
     ABT_xstream *bw_worker_xstreams = NULL;
+    ABT_sched *bw_worker_scheds = NULL;
+
+    /* NOTE: Margo is very likely to create a single producer (the
+     * progress function), multiple consumer usage pattern that
+     * causes excess memory consumption in some versions of
+     * Argobots.  See
+     * https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details.
+     * We therefore manually set the ABT_MEM_MAX_NUM_STACKS parameter 
+     * for Argobots to a low value so that RPC handler threads do not
+     * queue large numbers of stacks for reuse in per-ES data 
+     * structures.
+     */
+    putenv("ABT_MEM_MAX_NUM_STACKS=8");
 
     ABT_init(argc, argv);
     MPI_Init(&argc, &argv);
@@ -124,6 +134,37 @@ int main(int argc, char **argv)
         return(-1);
     }
 
+    /* boilerplate ABT initialization steps */
+    /****************************************/
+
+    /* get main pool for running mercury progress and RPC handlers */
+    /* NOTE: we use the ABT scheduler that idles while not busy */
+    ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 0, NULL,
+        ABT_SCHED_CONFIG_NULL, &sched);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_sched_create_basic()\n");
+        return(-1);
+    }
+    ret = ABT_xstream_self(&xstream);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_xstream_self()\n");
+        return(-1);
+    }
+    ret = ABT_xstream_set_main_sched(xstream, sched);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_xstream_set_main_sched()\n");
+        return(-1);
+    }
+    ret = ABT_xstream_get_main_pools(xstream, 1, &pool);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n");
+        return(-1);
+    }
+
     /* boilerplate HG initialization steps */
     /***************************************/
 
@@ -159,37 +200,6 @@ int main(int argc, char **argv)
         return(-1);
     }
 
-    if((rank == 0 && g_opts.snoozer_flag_client) || 
-        (rank == 1 && g_opts.snoozer_flag_server))
-    {
-#ifdef HAVE_ABT_SNOOZER
-        /* set primary ES to idle without polling in scheduler */
-        ret = ABT_snoozer_xstream_self_set();
-        if(ret != 0)
-        {
-            fprintf(stderr, "Error: ABT_snoozer_xstream_self_set()\n");
-            return(-1);
-        }
-#else
-        fprintf(stderr, "Error: abt-snoozer scheduler is not supported\n");
-        return(-1);
-#endif
-    }
-
-    /* get main pool for running mercury progress and RPC handlers */
-    ret = ABT_xstream_self(&xstream);
-    if(ret != 0)
-    {
-        fprintf(stderr, "Error: ABT_xstream_self()\n");
-        return(-1);
-    }   
-    ret = ABT_xstream_get_main_pools(xstream, 1, &pool);
-    if(ret != 0)
-    {
-        fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n");
-        return(-1);
-    }
-
     /* actually start margo */
     mid = margo_init_pool(pool, pool, hg_context);
     assert(mid);
@@ -242,10 +252,21 @@ int main(int argc, char **argv)
             /* run bulk transfers from a dedicated pool */
             bw_worker_xstreams = malloc(
                     g_opts.threads * sizeof(*bw_worker_xstreams));
-            assert(bw_worker_xstreams);
-            ret = ABT_snoozer_xstream_create(g_opts.threads, &g_transfer_pool,
-                    bw_worker_xstreams);
+            bw_worker_scheds = malloc(
+                    g_opts.threads * sizeof(*bw_worker_scheds));
+            assert(bw_worker_xstreams && bw_worker_scheds);
+
+            ret = ABT_pool_create_basic(ABT_POOL_FIFO_WAIT, ABT_POOL_ACCESS_MPMC,
+                    ABT_TRUE, &g_transfer_pool);
             assert(ret == ABT_SUCCESS);
+            for(i = 0; i < g_opts.threads; i++)
+            {
+                ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 1, &g_transfer_pool,
+                        ABT_SCHED_CONFIG_NULL, &bw_worker_scheds[i]);
+                assert(ret == ABT_SUCCESS);
+                ret = ABT_xstream_create(bw_worker_scheds[i], &bw_worker_xstreams[i]);
+                assert(ret == ABT_SUCCESS);
+            }
         }
 
         /* signaling mechanism for server to exit at conclusion of test */
@@ -275,6 +296,8 @@ int main(int argc, char **argv)
         }
         if(bw_worker_xstreams)
             free(bw_worker_xstreams);
+        if(bw_worker_scheds)
+            free(bw_worker_scheds);
     
         margo_bulk_free(g_bulk_handle);
     }
@@ -300,25 +323,16 @@ static void parse_args(int argc, char **argv, struct options *opts)
 {
     int opt;
     int ret;
-    char clientflag, serverflag;
 
     memset(opts, 0, sizeof(*opts));
 
     opts->concurrency = 1;
 
-#ifdef HAVE_ABT_SNOOZER
-    /* default to enabling snoozer scheduler on both client and server */
-    opts->snoozer_flag_client = 1;
-    opts->snoozer_flag_server = 1;
-#else
-    opts->snoozer_flag_client = 0;
-    opts->snoozer_flag_server = 0;
-#endif
     /* default to using whatever the standard timeout is in margo */
     opts->mercury_timeout_client = UINT_MAX;
     opts->mercury_timeout_server = UINT_MAX; 
 
-    while((opt = getopt(argc, argv, "n:x:c:T:d:s:t:D:")) != -1)
+    while((opt = getopt(argc, argv, "n:x:c:T:d:t:D:")) != -1)
     {
         switch(opt)
         {
@@ -362,18 +376,6 @@ static void parse_args(int argc, char **argv, struct options *opts)
                     exit(EXIT_FAILURE);
                 }
                 break;
-            case 's':
-                ret = sscanf(optarg, "%c,%c", &clientflag, &serverflag);
-                if(ret != 2)
-                {
-                    usage();
-                    exit(EXIT_FAILURE);
-                }
-                if(clientflag == '0') opts->snoozer_flag_client = 0;
-                else if(clientflag == '1') opts->snoozer_flag_client = 1;
-                if(serverflag == '0') opts->snoozer_flag_server = 0;
-                else if(serverflag == '1') opts->snoozer_flag_server = 1;
-                break;
             case 't':
                 ret = sscanf(optarg, "%u,%u", &opts->mercury_timeout_client, &opts->mercury_timeout_server);
                 if(ret != 2)
@@ -416,8 +418,6 @@ static void usage(void)
         "\t[-c concurrency] - number of concurrent operations to issue with ULTs\n"
         "\t[-T <os threads] - number of dedicated operating system threads to run ULTs on\n"
         "\t[-d filename] - enable diagnostics output \n"
-        "\t[-s <bool,bool>] - specify if snoozer scheduler is used on client and server\n"
-        "\t\t(e.g., -s 0,1 means snoozer disabled on client and enabled on server)\n"
         "\t\texample: mpiexec -n 2 ./margo-p2p-bw -x 4096 -D 30 -n verbs://\n"
         "\t\t(must be run with exactly 2 processes\n");
     

tests/perf-regression/margo-p2p-latency.c

@@ -14,9 +14,6 @@
 #include <mpi.h>
 
 #include <margo.h>
-#ifdef HAVE_ABT_SNOOZER
-#include <abt-snoozer.h>
-#endif
 #include <mercury.h>
 #include <abt.h>
 #include <ssg.h>
@@ -25,8 +22,6 @@
 struct options
 {
     int iterations;
-    int snoozer_flag_client;
-    int snoozer_flag_server;
     unsigned int mercury_timeout_client;
     unsigned int mercury_timeout_server;
     char* diag_file_name;
@@ -54,6 +49,7 @@ int main(int argc, char **argv)
     hg_context_t *hg_context;
     hg_class_t *hg_class;
     ABT_xstream xstream;
+    ABT_sched sched;
     ABT_pool pool;
     int ret;
     ssg_group_id_t gid;
@@ -63,6 +59,18 @@ int main(int argc, char **argv)
     int namelen;
     char processor_name[MPI_MAX_PROCESSOR_NAME];
 
+    /* NOTE: Margo is very likely to create a single producer (the
+     * progress function), multiple consumer usage pattern that
+     * causes excess memory consumption in some versions of
+     * Argobots.  See
+     * https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details.
+     * We therefore manually set the ABT_MEM_MAX_NUM_STACKS parameter 
+     * for Argobots to a low value so that RPC handler threads do not
+     * queue large numbers of stacks for reuse in per-ES data 
+     * structures.
+     */
+    putenv("ABT_MEM_MAX_NUM_STACKS=8");
+
     ABT_init(argc, argv);
     MPI_Init(&argc, &argv);
 
@@ -80,6 +88,37 @@ int main(int argc, char **argv)
 
     parse_args(argc, argv, &g_opts);
 
+    /* boilerplate ABT initialization steps */
+    /****************************************/
+
+    /* get main pool for running mercury progress and RPC handlers */
+    /* NOTE: we use the ABT scheduler that idles while not busy */
+    ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 0, NULL,
+        ABT_SCHED_CONFIG_NULL, &sched);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_sched_create_basic()\n");
+        return(-1);
+    }
+    ret = ABT_xstream_self(&xstream);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_xstream_self()\n");
+        return(-1);
+    }
+    ret = ABT_xstream_set_main_sched(xstream, sched);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_xstream_set_main_sched()\n");
+        return(-1);
+    }
+    ret = ABT_xstream_get_main_pools(xstream, 1, &pool);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n");
+        return(-1);
+    }
+
     /* boilerplate HG initialization steps */
     /***************************************/
 
@@ -115,37 +154,6 @@ int main(int argc, char **argv)
         return(-1);
     }
 
-    if((rank == 0 && g_opts.snoozer_flag_client) || 
-        (rank == 1 && g_opts.snoozer_flag_server))
-    {
-#ifdef HAVE_ABT_SNOOZER
-        /* set primary ES to idle without polling in scheduler */
-        ret = ABT_snoozer_xstream_self_set();
-        if(ret != 0)
-        {
-            fprintf(stderr, "Error: ABT_snoozer_xstream_self_set()\n");
-            return(-1);
-        }
-#else
-        fprintf(stderr, "Error: abt-snoozer scheduler is not supported\n");
-        return(-1);
-#endif
-    }
-
-    /* get main pool for running mercury progress and RPC handlers */
-    ret = ABT_xstream_self(&xstream);
-    if(ret != 0)
-    {
-        fprintf(stderr, "Error: ABT_xstream_self()\n");
-        return(-1);
-    }   
-    ret = ABT_xstream_get_main_pools(xstream, 1, &pool);
-    if(ret != 0)
-    {
-        fprintf(stderr, "Error: ABT_xstream_get_main_pools()\n");
-        return(-1);
-    }
-
     /* actually start margo */
     mid = margo_init_pool(pool, pool, hg_context);
     assert(mid);
@@ -226,23 +234,14 @@ static void parse_args(int argc, char **argv, struct options *opts)
 {
     int opt;
     int ret;
-    char clientflag, serverflag;
 
     memset(opts, 0, sizeof(*opts));
 
-#ifdef HAVE_ABT_SNOOZER
-    /* default to enabling snoozer scheduler on both client and server */
-    opts->snoozer_flag_client = 1;
-    opts->snoozer_flag_server = 1;
-#else
-    opts->snoozer_flag_client = 0;
-    opts->snoozer_flag_server = 0;
-#endif
     /* default to using whatever the standard timeout is in margo */
     opts->mercury_timeout_client = UINT_MAX;
     opts->mercury_timeout_server = UINT_MAX; 
 
-    while((opt = getopt(argc, argv, "n:i:d:s:t:")) != -1)
+    while((opt = getopt(argc, argv, "n:i:d:t:")) != -1)
     {
         switch(opt)
         {
@@ -262,18 +261,6 @@ static void parse_args(int argc, char **argv, struct options *opts)
                     exit(EXIT_FAILURE);
                 }
                 break;
-            case 's':
-                ret = sscanf(optarg, "%c,%c", &clientflag, &serverflag);
-                if(ret != 2)
-                {
-                    usage();
-                    exit(EXIT_FAILURE);
-                }
-                if(clientflag == '0') opts->snoozer_flag_client = 0;
-                else if(clientflag == '1') opts->snoozer_flag_client = 1;
-                if(serverflag == '0') opts->snoozer_flag_server = 0;
-                else if(serverflag == '1') opts->snoozer_flag_server = 1;
-                break;
             case 't':
                 ret = sscanf(optarg, "%u,%u", &opts->mercury_timeout_client, &opts->mercury_timeout_server);
                 if(ret != 2)
@@ -313,8 +300,6 @@ static void usage(void)
         "\t-i <iterations> - number of RPC iterations\n"
         "\t-n <na> - na transport\n"
         "\t[-d filename] - enable diagnostics output \n"
-        "\t[-s <bool,bool>] - specify if snoozer scheduler is used on client and server\n"
-        "\t\t(e.g., -s 0,1 means snoozer disabled on client and enabled on server)\n"
         "\t\texample: mpiexec -n 2 ./margo-p2p-latency -i 10000 -n verbs://\n"
         "\t\t(must be run with exactly 2 processes\n");
     

tests/perf-regression/theta/run-regression-static.sh

@@ -38,7 +38,6 @@ git clone https://github.com/carns/argobots.git
 git clone https://github.com/mercury-hpc/mercury.git
 wget http://dist.schmorp.de/libev/libev-4.24.tar.gz
 tar -xvzf libev-4.24.tar.gz
-git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git
 git clone https://xgitlab.cels.anl.gov/sds/margo.git
 git clone https://xgitlab.cels.anl.gov/sds/ssg.git
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
@@ -111,16 +110,6 @@ cd build
 make -j 3
 make install
 
-# abt-snoozer
-echo "=== BUILDING ABT-SNOOZER ==="
-cd $SANDBOX/abt-snoozer
-./prepare.sh
-mkdir build
-cd build
-../configure --prefix=$PREFIX --enable-shared=no  --host=x86_64-linux
-make -j 3
-make install
-
 # margo
 echo "=== BUILDING MARGO ==="
 cd $SANDBOX/margo

tests/perf-regression/theta/run-regression.sh

@@ -40,7 +40,6 @@ git clone https://github.com/carns/argobots.git
 git clone https://github.com/mercury-hpc/mercury.git
 wget http://dist.schmorp.de/libev/libev-4.24.tar.gz
 tar -xvzf libev-4.24.tar.gz
-git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer.git
 git clone https://xgitlab.cels.anl.gov/sds/margo.git
 git clone https://xgitlab.cels.anl.gov/sds/ssg.git
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.3.2.tar.gz
@@ -106,16 +105,6 @@ cd build
 make -j 3
 make install
 
-# abt-snoozer
-echo "=== BUILDING ABT-SNOOZER ==="
-cd $SANDBOX/abt-snoozer
-./prepare.sh
-mkdir build
-cd build
-../configure --prefix=$PREFIX  --host=x86_64-linux
-make -j 3
-make install
-
 # margo
 echo "=== BUILDING MARGO ==="
 cd $SANDBOX/margo