Commit 941c3388 authored by Philip Carns's avatar Philip Carns

use BASIC_WAIT scheduler instead of abt-snoozer

parent d4e6841a
...@@ -26,9 +26,7 @@ Note that Margo should be compatible with any Mercury transport (NA plugin). Th ...@@ -26,9 +26,7 @@ Note that Margo should be compatible with any Mercury transport (NA plugin). Th
## Dependencies ## Dependencies
* mercury (git clone --recurse-submodules https://github.com/mercury-hpc/mercury.git) * mercury (git clone --recurse-submodules https://github.com/mercury-hpc/mercury.git)
* argobots (git clone https://github.com/pmodels/argobots.git) * argobots fork with scheduler wait support (git -b dev-fifo-wait clone https://github.com/carns/argobots.git)
* libev (e.g libev-dev package on Ubuntu or Debian)
* (optional) abt-snoozer (git clone https://xgitlab.cels.anl.gov/sds/abt-snoozer)
### Recommended Mercury build options ### Recommended Mercury build options
......
...@@ -73,33 +73,7 @@ CPPFLAGS="$ARGOBOTS_CFLAGS $CPPFLAGS" ...@@ -73,33 +73,7 @@ CPPFLAGS="$ARGOBOTS_CFLAGS $CPPFLAGS"
CFLAGS="$ARGOBOTS_CFLAGS $CFLAGS" CFLAGS="$ARGOBOTS_CFLAGS $CFLAGS"
PC_REQUIRES="mercury argobots" PC_REQUIRES="mercury argobots"
AC_SUBST([PC_REQUIRES], ["$PC_REQUIRES"])
check_abt_snoozer=auto
AC_ARG_ENABLE([abt-snoozer],
[ --enable-abt-snoozer enable abt-snoozer (default: dynamic check)],
[ case "${enableval}" in
yes) check_abt_snoozer=yes ;;
no) check_abt_snoozer=no ;;
*) AC_MSG_ERROR([bad value ${enableval} for --enable-abt-snoozer]) ;;
esac],
[])
check_abt_snoozer_status=fail
if test "x${check_abt_snoozer}" = xauto -o "x${check_abt_snoozer}" = xyes ; then
PKG_CHECK_MODULES([ABT_SNOOZER],[abt-snoozer],
[AC_DEFINE([HAVE_ABT_SNOOZER], [1], [Use ABT-SNOOZER])
PC_REQUIRES="$PC_REQUIRES abt-snoozer"
LIBS="$ABT_SNOOZER_LIBS $LIBS"
CPPFLAGS="$ABT_SNOOZER_CFLAGS $CPPFLAGS"
CFLAGS="$ABT_SNOOZER_CFLAGS $CFLAGS"
AC_SUBST([PC_REQUIRES], ["$PC_REQUIRES"])
check_abt_snoozer_status=success],
[printf ""])
fi
if test "x${check_abt_snoozer_status}" = xfail -a "x${check_abt_snoozer}" = xyes; then
AC_MSG_ERROR([Could not find working abt-snoozer installation!])
fi
# turn off a warning that is tripping up our use of utlist at the moment # turn off a warning that is tripping up our use of utlist at the moment
CFLAGS="-Wno-address $CFLAGS" CFLAGS="-Wno-address $CFLAGS"
...@@ -118,6 +92,18 @@ AC_MSG_RESULT(no) ...@@ -118,6 +92,18 @@ AC_MSG_RESULT(no)
AC_MSG_ERROR([Boost preprocessing not enabled in Mercury]) AC_MSG_ERROR([Boost preprocessing not enabled in Mercury])
) )
# make sure that argobots has the ABT_SCHED_BASIC_WAIT scheduler
AC_MSG_CHECKING(for ABT_SCHED_BASIC_WAIT in Argobots)
AC_TRY_COMPILE([
#include <abt.h>
], [
enum ABT_sched_predef sched = ABT_SCHED_BASIC_WAIT;
],
AC_MSG_RESULT(yes),
AC_MSG_RESULT(no)
AC_MSG_ERROR([ABT_SCHED_BASIC_WAIT not available in Argobots])
)
AC_CONFIG_FILES([Makefile maint/margo.pc]) AC_CONFIG_FILES([Makefile maint/margo.pc])
AC_OUTPUT AC_OUTPUT
...@@ -12,9 +12,6 @@ ...@@ -12,9 +12,6 @@
#include <stdlib.h> #include <stdlib.h>
#include <margo-config.h> #include <margo-config.h>
#ifdef HAVE_ABT_SNOOZER
#include <abt-snoozer.h>
#endif
#include <time.h> #include <time.h>
#include <math.h> #include <math.h>
...@@ -179,7 +176,11 @@ margo_instance_id margo_init(const char *addr_str, int mode, ...@@ -179,7 +176,11 @@ margo_instance_id margo_init(const char *addr_str, int mode,
{ {
ABT_xstream progress_xstream = ABT_XSTREAM_NULL; ABT_xstream progress_xstream = ABT_XSTREAM_NULL;
ABT_pool progress_pool = ABT_POOL_NULL; ABT_pool progress_pool = ABT_POOL_NULL;
ABT_sched progress_sched;
ABT_sched self_sched;
ABT_xstream self_xstream;
ABT_xstream *rpc_xstreams = NULL; ABT_xstream *rpc_xstreams = NULL;
ABT_sched *rpc_scheds = NULL;
ABT_xstream rpc_xstream = ABT_XSTREAM_NULL; ABT_xstream rpc_xstream = ABT_XSTREAM_NULL;
ABT_pool rpc_pool = ABT_POOL_NULL; ABT_pool rpc_pool = ABT_POOL_NULL;
hg_class_t *hg_class = NULL; hg_class_t *hg_class = NULL;
...@@ -211,49 +212,49 @@ margo_instance_id margo_init(const char *addr_str, int mode, ...@@ -211,49 +212,49 @@ margo_instance_id margo_init(const char *addr_str, int mode,
abt_init = 1; abt_init = 1;
} }
/* set caller (self) ES to idle without polling */ /* set caller (self) ES to sleep when idle by using sched_wait */
#ifdef HAVE_ABT_SNOOZER ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 0, NULL,
ret = ABT_snoozer_xstream_self_set(); ABT_SCHED_CONFIG_NULL, &self_sched);
if(ret != 0) goto err; if(ret != ABT_SUCCESS) goto err;
#endif ret = ABT_xstream_self(&self_xstream);
if(ret != ABT_SUCCESS) goto err;
ret = ABT_xstream_set_main_sched(self_xstream, self_sched);
if(ret != ABT_SUCCESS) goto err;
if (use_progress_thread) if (use_progress_thread)
{ {
#ifdef HAVE_ABT_SNOOZER /* create an xstream to run progress engine */
ret = ABT_snoozer_xstream_create(1, &progress_pool, &progress_xstream); ret = ABT_sched_create_basic(ABT_SCHED_BASIC_WAIT, 0, NULL,
if (ret != ABT_SUCCESS) goto err; ABT_SCHED_CONFIG_NULL, &progress_sched);
#else if (ret != ABT_SUCCESS) goto err;
ret = ABT_xstream_create(ABT_SCHED_NULL, &progress_xstream); ret = ABT_xstream_create(progress_sched, &progress_xstream);
if (ret != ABT_SUCCESS) goto err; if (ret != ABT_SUCCESS) goto err;
ret = ABT_xstream_get_main_pools(progress_xstream, 1, &progress_pool);
if (ret != ABT_SUCCESS) goto err;
#endif
} }
else else
{ {
ret = ABT_xstream_self(&progress_xstream); ret = ABT_xstream_self(&progress_xstream);
if (ret != ABT_SUCCESS) goto err; if (ret != ABT_SUCCESS) goto err;
ret = ABT_xstream_get_main_pools(progress_xstream, 1, &progress_pool);
if (ret != ABT_SUCCESS) goto err;
} }
ret = ABT_xstream_get_main_pools(progress_xstream, 1, &progress_pool);
if (ret != ABT_SUCCESS) goto err;
if (rpc_thread_count > 0) if (rpc_thread_count > 0)
{ {
/* create a collection of xstreams to run RPCs */
rpc_xstreams = calloc(rpc_thread_count, sizeof(*rpc_xstreams)); rpc_xstreams = calloc(rpc_thread_count, sizeof(*rpc_xstreams));
if (rpc_xstreams == NULL) goto err; if (rpc_xstreams == NULL) goto err;
#ifdef HAVE_ABT_SNOOZER rpc_scheds = calloc(rpc_thread_count, sizeof(*rpc_scheds));
ret = ABT_snoozer_xstream_create(rpc_thread_count, &rpc_pool, if (rpc_scheds == NULL) goto err;
rpc_xstreams); ret = ABT_pool_create_basic(ABT_POOL_FIFO_WAIT, ABT_POOL_ACCESS_MPMC, ABT_TRUE, &rpc_pool);
if (ret != ABT_SUCCESS) goto err; if (ret != ABT_SUCCESS) goto err;
#else for(i=0; i<rpc_thread_count; i++)
int j; {
ret = ABT_pool_create_basic(ABT_POOL_FIFO, ABT_POOL_ACCESS_MPMC, ABT_TRUE, &rpc_pool); ret = ABT_sched_create_basic(ABT_SCHED_BASIC, 1, &rpc_pool,
if (ret != ABT_SUCCESS) goto err; ABT_SCHED_CONFIG_NULL, &rpc_scheds[i]);
for(j=0; j<rpc_thread_count; j++) { if (ret != ABT_SUCCESS) goto err;
ret = ABT_xstream_create(ABT_SCHED_NULL, rpc_xstreams+j); ret = ABT_xstream_create(rpc_scheds[i], rpc_xstreams+i);
if (ret != ABT_SUCCESS) goto err; if (ret != ABT_SUCCESS) goto err;
} }
#endif
} }
else if (rpc_thread_count == 0) else if (rpc_thread_count == 0)
{ {
...@@ -306,6 +307,7 @@ err: ...@@ -306,6 +307,7 @@ err:
ABT_xstream_free(&rpc_xstreams[i]); ABT_xstream_free(&rpc_xstreams[i]);
} }
free(rpc_xstreams); free(rpc_xstreams);
free(rpc_scheds);
} }
if(hg_context) if(hg_context)
HG_Context_destroy(hg_context); HG_Context_destroy(hg_context);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment