limit size of abt stack cache

- See #40

include/margo.h

@@ -74,6 +74,16 @@ margo_instance_id margo_init(
  * @param [in] handler_pool Argobots pool to service RPC handlers
  * @param [in] hg_context Mercury context
  * @returns margo instance id on success, MARGO_INSTANCE_NULL upon error
+ *
+ * NOTE: if you are configuring Argobots pools yourself before
+ * passing them into this function, please consider setting
+ * ABT_MEM_MAX_NUM_STACKS to a low value (like 8) either in your
+ * environment or programmatically with putenv() in your code before
+ * creating the pools to prevent excess memory consumption under
+ * load from producer/consumer patterns across execution streams that
+ * fail to utilize per-execution stream stack caches.  See
+ * https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details.
+ * The margo_init() function does this automatically.
  */
 margo_instance_id margo_init_pool(
     ABT_pool progress_pool,

src/margo.c

@@ -9,6 +9,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <abt.h>
+#include <stdlib.h>
 
 #include <margo-config.h>
 #ifdef HAVE_ABT_SNOOZER
@@ -167,6 +168,18 @@ margo_instance_id margo_init(const char *addr_str, int mode,
 
     if(mode != MARGO_CLIENT_MODE && mode != MARGO_SERVER_MODE) goto err;
 
+    /* NOTE: Margo is very likely to create a single producer (the
+     * progress function), multiple consumer usage pattern that
+     * causes excess memory consumption in some versions of
+     * Argobots.  See
+     * https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details.
+     * We therefore manually set the ABT_MEM_MAX_NUM_STACKS parameter 
+     * for Argobots to a low value so that RPC handler threads do not
+     * queue large numbers of stacks for reuse in per-ES data 
+     * structures.
+     */
+    putenv("ABT_MEM_MAX_NUM_STACKS=8");
+
     if (ABT_initialized() == ABT_ERR_UNINITIALIZED)
     {
         ret = ABT_init(0, NULL); /* XXX: argc/argv not currently used by ABT ... */