Commit d49d5e00 authored by Philip Carns's avatar Philip Carns

limit size of abt stack cache

- See #40
parent f16e0a0b
...@@ -74,6 +74,16 @@ margo_instance_id margo_init( ...@@ -74,6 +74,16 @@ margo_instance_id margo_init(
* @param [in] handler_pool Argobots pool to service RPC handlers * @param [in] handler_pool Argobots pool to service RPC handlers
* @param [in] hg_context Mercury context * @param [in] hg_context Mercury context
* @returns margo instance id on success, MARGO_INSTANCE_NULL upon error * @returns margo instance id on success, MARGO_INSTANCE_NULL upon error
*
* NOTE: if you are configuring Argobots pools yourself before
* passing them into this function, please consider setting
* ABT_MEM_MAX_NUM_STACKS to a low value (like 8) either in your
* environment or programmatically with putenv() in your code before
* creating the pools to prevent excess memory consumption under
* load from producer/consumer patterns across execution streams that
* fail to utilize per-execution stream stack caches. See
* https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details.
* The margo_init() function does this automatically.
*/ */
margo_instance_id margo_init_pool( margo_instance_id margo_init_pool(
ABT_pool progress_pool, ABT_pool progress_pool,
......
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
#include <unistd.h> #include <unistd.h>
#include <errno.h> #include <errno.h>
#include <abt.h> #include <abt.h>
#include <stdlib.h>
#include <margo-config.h> #include <margo-config.h>
#ifdef HAVE_ABT_SNOOZER #ifdef HAVE_ABT_SNOOZER
...@@ -167,6 +168,18 @@ margo_instance_id margo_init(const char *addr_str, int mode, ...@@ -167,6 +168,18 @@ margo_instance_id margo_init(const char *addr_str, int mode,
if(mode != MARGO_CLIENT_MODE && mode != MARGO_SERVER_MODE) goto err; if(mode != MARGO_CLIENT_MODE && mode != MARGO_SERVER_MODE) goto err;
/* NOTE: Margo is very likely to create a single producer (the
* progress function), multiple consumer usage pattern that
* causes excess memory consumption in some versions of
* Argobots. See
* https://xgitlab.cels.anl.gov/sds/margo/issues/40 for details.
* We therefore manually set the ABT_MEM_MAX_NUM_STACKS parameter
* for Argobots to a low value so that RPC handler threads do not
* queue large numbers of stacks for reuse in per-ES data
* structures.
*/
putenv("ABT_MEM_MAX_NUM_STACKS=8");
if (ABT_initialized() == ABT_ERR_UNINITIALIZED) if (ABT_initialized() == ABT_ERR_UNINITIALIZED)
{ {
ret = ABT_init(0, NULL); /* XXX: argc/argv not currently used by ABT ... */ ret = ABT_init(0, NULL); /* XXX: argc/argv not currently used by ABT ... */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment