Merge branch 'dfp-big-to-release' into 'master'

Dragonfly Plus and Related Work

See merge request !53

Makefile.am

@@ -13,6 +13,8 @@ EXTRA_PROGRAMS =
 CLEANFILES = $(bin_SCRIPTS)
 EXTRA_DIST =
 BUILT_SOURCES =
+AM_LDFLAGS =
+
 
 # pkgconfig files
 pkgconfigdir = $(libdir)/pkgconfig
@@ -57,6 +59,12 @@ AM_CPPFLAGS += ${RECORDER_CPPFLAGS}
 src_libcodes_la_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
 endif
 
+if USE_ONLINE
+AM_CPPFLAGS += ${ARGOBOTS_CFLAGS} ${SWM_CFLAGS} -DUSE_ONLINE=1
+LDADD += ${SWM_LIBS} ${ARGOBOTS_LIBS}
+src_libcodes_la_SOURCES += src/workload/methods/codes-online-comm-wrkld.C
+endif
+
 if USE_DUMPI
 AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
 src_libcodes_la_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c

codes/codes-workload.h

@@ -19,6 +19,9 @@ extern "C" {
 #include <ross.h>
 #include "configuration.h"
 
+#ifdef USE_ONLINE
+#include <abt.h>
+#endif
 #define MAX_NAME_LENGTH_WKLD 512
 
 /* implementations included with codes */
@@ -30,6 +33,7 @@ typedef struct recorder_params recorder_params;
 /* struct to hold the actual data from a single MPI event*/
 typedef struct dumpi_trace_params dumpi_trace_params;
 typedef struct checkpoint_wrkld_params checkpoint_wrkld_params;
+typedef struct online_comm_params online_comm_params;
 
 struct iomock_params
 {
@@ -77,6 +81,11 @@ struct dumpi_trace_params {
 #endif
 };
 
+struct online_comm_params {
+    char workload_name[MAX_NAME_LENGTH_WKLD];
+    char file_path[MAX_NAME_LENGTH_WKLD];
+    int nprocs;
+};
 struct checkpoint_wrkld_params
 {
     int nprocs; /* number of workload processes */
@@ -235,14 +244,14 @@ struct codes_workload_op
         } collective;
         struct {
             int count;
-            unsigned int* req_ids;
+            uint32_t* req_ids;
         } waits;
         struct {
-            unsigned int req_id;
+            uint32_t req_id;
         } wait;
         struct
         {
-            unsigned int req_id;
+            uint32_t req_id;
         }
         free;
     }u;
@@ -323,6 +332,13 @@ int codes_workload_get_rank_cnt(
         const char* params,
         int app_id);
 
+/* Finalize the workload */
+int codes_workload_finalize(
+        const char* type,
+        const char* params,
+        int app_id, 
+        int rank);
+
 /* for debugging/logging: print an individual operation to the specified file */
 void codes_workload_print_op(
         FILE *f,
@@ -346,6 +362,7 @@ struct codes_workload_method
     void (*codes_workload_get_next)(int app_id, int rank, struct codes_workload_op *op);
     void (*codes_workload_get_next_rc2)(int app_id, int rank);
     int (*codes_workload_get_rank_cnt)(const char* params, int app_id);
+    int (*codes_workload_finalize)(const char* params, int app_id, int rank);
     /* added for get all read or write time */
     int (*codes_workload_get_time)(const char * params, int app_id, int rank, double *read_time, double *write_time, int64_t *read_bytes, int64_t *written_bytes);
 };

codes/model-net-lp.h

@@ -6,11 +6,11 @@
 
 /* This is the base model-net LP that all events pass through before
  * performing any topology-specific work. Packet scheduling, dealing with
- * packet loss (potentially), etc. happens here. 
- * Additionally includes wrapper event "send" function that all 
+ * packet loss (potentially), etc. happens here.
+ * Additionally includes wrapper event "send" function that all
  * events for underlying models must go through */
 
-#ifndef MODEL_NET_LP_H  
+#ifndef MODEL_NET_LP_H
 #define MODEL_NET_LP_H
 
 #ifdef __cplusplus
@@ -23,6 +23,7 @@ extern "C" {
 #include "model-net-sched.h"
 #include "net/dragonfly.h"
 #include "net/dragonfly-custom.h"
+#include "net/dragonfly-plus.h"
 #include "net/slimfly.h"
 #include "net/fattree.h"
 #include "net/loggp.h"
@@ -43,10 +44,10 @@ void model_net_base_configure();
 /// model developers
 
 // Construct a model-net-specific event, analagous to a tw_event_new and
-// codes_event_new. The difference here is that we return pointers to 
+// codes_event_new. The difference here is that we return pointers to
 // both the message data (to be cast into the appropriate type) and the
 // pointer to the end of the event struct.
-// 
+//
 // This function is expected to be called within each specific model-net
 // method - strange and disturbing things will happen otherwise
 tw_event * model_net_method_event_new(
@@ -131,6 +132,7 @@ typedef struct model_net_wrap_msg {
         model_net_base_msg      m_base;  // base lp
         terminal_message        m_dfly;  // dragonfly
         terminal_custom_message        m_custom_dfly;  // dragonfly-custom
+        terminal_plus_message        m_dfly_plus;  // dragonfly plus
         slim_terminal_message   m_slim;  // slimfly
 	fattree_message		m_fat;   // fattree
         loggp_message           m_loggp; // loggp

codes/model-net.h

@@ -41,7 +41,7 @@ extern "C" {
 /* HACK: there is currently no scheduling fidelity across multiple
  * model_net_event calls. Hence, problems arise when some LP sends multiple
  * messages as part of an event and expects FCFS ordering. A proper fix which
- * involves model-net LP-level scheduling of requests is ideal, but not 
+ * involves model-net LP-level scheduling of requests is ideal, but not
  * feasible for now (would basically have to redesign model-net), so expose
  * explicit start-sequence and stop-sequence markers as a workaround
  */
@@ -74,6 +74,8 @@ typedef struct mn_stats mn_stats;
     X(LOGGP,     "modelnet_loggp",     "loggp",     &loggp_method)\
     X(EXPRESS_MESH, "modelnet_express_mesh", "express_mesh", &express_mesh_method)\
     X(EXPRESS_MESH_ROUTER, "modelnet_express_mesh_router", "express_mesh_router", &express_mesh_router_method)\
+    X(DRAGONFLY_PLUS, "modelnet_dragonfly_plus", "dragonfly_plus", &dragonfly_plus_method)\
+    X(DRAGONFLY_PLUS_ROUTER, "modelnet_dragonfly_plus_router", "dragonfly_plus_router", &dragonfly_plus_router_method)\
     X(MAX_NETS,  NULL,                 NULL,        NULL)
 
 #define X(a,b,c,d) a,
@@ -144,7 +146,7 @@ struct mn_stats
     long max_event_size;
 };
 
-/* Registers all model-net LPs in ROSS. Should be called after 
+/* Registers all model-net LPs in ROSS. Should be called after
  * configuration_load, but before codes_mapping_setup */
 void model_net_register();
 
@@ -166,8 +168,8 @@ void model_net_enable_sampling(tw_stime interval, tw_stime end);
 int model_net_sampling_enabled(void);
 
 /* Initialize/configure the network(s) based on the CODES configuration.
- * returns an array of the network ids, indexed in the order given by the 
- * modelnet_order configuration parameter 
+ * returns an array of the network ids, indexed in the order given by the
+ * modelnet_order configuration parameter
  * OUTPUT id_count - the output number of networks */
 int* model_net_set_params(int *id_count);
 
@@ -189,7 +191,7 @@ void model_net_event_collective_rc(
         int message_size,
         tw_lp *sender);
 
-/* allocate and transmit a new event that will pass through model_net to 
+/* allocate and transmit a new event that will pass through model_net to
  * arrive at its destination:
  *
  * - net_id: the type of network to send this message through. The set of
@@ -231,9 +233,9 @@ void model_net_event_collective_rc(
 // first argument becomes the network ID
 model_net_event_return model_net_event(
     int net_id,
-    char const * category, 
-    tw_lpid final_dest_lp, 
-    uint64_t message_size, 
+    char const * category,
+    tw_lpid final_dest_lp,
+    uint64_t message_size,
     tw_stime offset,
     int remote_event_size,
     void const * remote_event,
@@ -251,9 +253,9 @@ model_net_event_return model_net_event(
 model_net_event_return model_net_event_annotated(
         int net_id,
         char const * annotation,
-        char const * category, 
-        tw_lpid final_dest_lp, 
-        uint64_t message_size, 
+        char const * category,
+        tw_lpid final_dest_lp,
+        uint64_t message_size,
         tw_stime offset,
         int remote_event_size,
         void const * remote_event,
@@ -270,9 +272,9 @@ model_net_event_return model_net_event_mctx(
         int net_id,
         struct codes_mctx const * send_map_ctx,
         struct codes_mctx const * recv_map_ctx,
-        char const * category, 
-        tw_lpid final_dest_lp, 
-        uint64_t message_size, 
+        char const * category,
+        tw_lpid final_dest_lp,
+        uint64_t message_size,
         tw_stime offset,
         int remote_event_size,
         void const * remote_event,
@@ -309,7 +311,7 @@ int model_net_get_msg_sz(int net_id);
  *   identical to the sender argument to tw_event_new().
  */
 /* NOTE: we may end up needing additoinal arguments here to track state for
- * reverse computation; add as needed 
+ * reverse computation; add as needed
  */
 DEPRECATED
 void model_net_event_rc(
@@ -333,7 +335,7 @@ void model_net_event_rc2(
  * Parameters are largely the same as model_net_event, with the following
  * exceptions:
  * - final_dest_lp is the lp to pull data from
- * - self_event_size, self_event are applied at the requester upon receipt of 
+ * - self_event_size, self_event are applied at the requester upon receipt of
  *   the payload from the dest
  */
 model_net_event_return model_net_pull_event(
@@ -383,7 +385,7 @@ void model_net_pull_event_rc(
  * model-net implementation (currently implemented as a set of translation-unit
  * globals). Upon a subsequent model_net_*event* call, the context is consumed
  * and reset to an unused state.
- * 
+ *
  * NOTE: this call MUST be placed in the same calling context as the subsequent
  * model_net_*event* call. Otherwise, the parameters are not guaranteed to work
  * on the intended event, and may possibly be consumed by another, unrelated

codes/net/dragonfly-plus.h

+/*
+ * Copyright (C) 2014 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef DRAGONFLY_PLUS_H
+#define DRAGONFLY_PLUS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <ross.h>
+
+typedef struct terminal_plus_message terminal_plus_message;
+
+/* this message is used for both dragonfly compute nodes and routers */
+struct terminal_plus_message
+{
+  /* magic number */
+  int magic;
+  /* flit travel start time*/
+  tw_stime travel_start_time;
+ /* packet ID of the flit  */
+  unsigned long long packet_ID;
+  /* event type of the flit */
+  short  type;
+  /* category: comes from codes */
+  char category[CATEGORY_NAME_MAX];
+  /* store category hash in the event */
+  uint32_t category_hash;
+  /* final destination LP ID, this comes from codes can be a server or any other LP type*/
+  tw_lpid final_dest_gid;
+  /*sending LP ID from CODES, can be a server or any other LP type */
+  tw_lpid sender_lp;
+  tw_lpid sender_mn_lp; // source modelnet id
+ /* destination terminal ID of the dragonfly */
+  tw_lpid dest_terminal_id;
+  /* source terminal ID of the dragonfly */
+  unsigned int src_terminal_id;
+  /* message originating router id. MM: Can we calculate it through
+   * sender_mn_lp??*/
+  unsigned int origin_router_id;
+
+  /* number of hops traversed by the packet */
+  short my_N_hop;
+  short my_l_hop, my_g_hop;
+  short saved_channel;
+  short saved_vc;
+
+  int next_stop;
+
+  short nonmin_done;
+  /* Intermediate LP ID from which this message is coming */
+  unsigned int intm_lp_id;
+  /* last hop of the message, can be a terminal, local router or global router */
+  short last_hop;
+   /* For routing */
+  int saved_src_dest;
+  int saved_src_chan;
+
+  //DFP Specific Routing
+  int intm_rtr_id; //Router ID of the intermediate router for nonminimal routes
+  int intm_group_id; //Group ID of the intermediate router for nonminimal routes
+
+  short dfp_upward_channel_flag;
+
+  int dfp_dest_terminal_id; //this is the terminal id in the dfp network in range [0-total_num_terminals)
+
+   uint32_t chunk_id;
+   uint32_t packet_size;
+   uint32_t message_id;
+   uint32_t total_size;
+
+   int remote_event_size_bytes;
+   int local_event_size_bytes;
+
+  // For buffer message
+   short vc_index;
+   int output_chan;
+   model_net_event_return event_rc;
+   int is_pull;
+   uint32_t pull_size;
+
+   /* for reverse computation */
+   int path_type;
+   tw_stime saved_available_time;
+   tw_stime saved_avg_time;
+   tw_stime saved_rcv_time;
+   tw_stime saved_busy_time;
+   tw_stime saved_total_time;
+   tw_stime saved_sample_time;
+   tw_stime msg_start_time;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* end of include guard: DRAGONFLY_H */
+
+/*
+ * Local variables:
+ *  c-indent-level: 4
+ *  c-basic-offset: 4
+ * End:
+ *
+ * vim: ft=c ts=8 sts=4 sw=4 expandtab
+ */

configure.ac

@@ -5,6 +5,10 @@ AC_PREREQ([2.67])
 AC_INIT([codes], [0.6.0], [http://trac.mcs.anl.gov/projects/codes/newticket],[],[http://www.mcs.anl.gov/projects/codes/])
 LT_INIT
 
+
+#WRAP SOME OPTION AROUND THIS - IT'S NOT REQUIRED FOR 99% OF CODES
+AX_BOOST_BASE([1.66])
+
 AC_CANONICAL_TARGET
 AC_CANONICAL_SYSTEM
 AC_CANONICAL_HOST
@@ -26,6 +30,7 @@ AC_PROG_CC
 AM_PROG_CC_C_O
 AC_PROG_CXX
 AC_PROG_CXXCPP
+AX_CXX_COMPILE_STDCXX(11, noext, mandatory)
 AC_PROG_RANLIB
 
 PKG_PROG_PKG_CONFIG
@@ -104,6 +109,24 @@ fi
 AM_CONDITIONAL(USE_DARSHAN, [test "x${use_darshan}" = xyes])
 
 
+# check for Argobots
+AC_ARG_WITH([online],[AS_HELP_STRING([--with-online@<:@=DIR@:>@],
+                        [Build with the online workloads and argobots support])],
+                      [use_online=yes],[use_online=no])
+if test "x${use_online}" != "xno" ; then
+    AM_CONDITIONAL(USE_ONLINE, true)
+    PKG_CHECK_MODULES_STATIC([ARGOBOTS], [argobots], [],
+                      [AC_MSG_ERROR([Could not find working argobots installation via pkg-config])])
+    PKG_CHECK_MODULES_STATIC([SWM], [swm], [],
+                      [AC_MSG_ERROR([Could not find working swm installation via pkg-config])])
+    PKG_CHECK_VAR([SWM_DATAROOTDIR], [swm], [datarootdir], [],
+              [AC_MSG_ERROR[Could not find shared directory in SWM]])
+    AC_DEFINE_UNQUOTED([SWM_DATAROOTDIR], ["$SWM_DATAROOTDIR"], [if using json
+                        data files])
+else
+    AM_CONDITIONAL(USE_ONLINE, false)
+fi
+
 # check for Recorder
 AM_CONDITIONAL(USE_RECORDER, true)
 RECORDER_CPPFLAGS="-DUSE_RECORDER=1"

m4/m4_ax_boost_base.m4

+# ===========================================================================
+#      https://www.gnu.org/software/autoconf-archive/ax_boost_base.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_BOOST_BASE([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+#
+# DESCRIPTION
+#
+#   Test for the Boost C++ libraries of a particular version (or newer)
+#
+#   If no path to the installed boost library is given the macro searchs
+#   under /usr, /usr/local, /opt and /opt/local and evaluates the
+#   $BOOST_ROOT environment variable. Further documentation is available at
+#   <http://randspringer.de/boost/index.html>.
+#
+#   This macro calls:
+#
+#     AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS)
+#
+#   And sets:
+#
+#     HAVE_BOOST
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
+#   Copyright (c) 2009 Peter Adolphs
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 44
+
+# example boost program (need to pass version)
+m4_define([_AX_BOOST_BASE_PROGRAM],
+          [AC_LANG_PROGRAM([[
+#include <boost/version.hpp>
+]],[[
+(void) ((void)sizeof(char[1 - 2*!!((BOOST_VERSION) < ($1))]));
+]])])
+
+AC_DEFUN([AX_BOOST_BASE],
+[
+AC_ARG_WITH([boost],
+  [AS_HELP_STRING([--with-boost@<:@=ARG@:>@],
+    [use Boost library from a standard location (ARG=yes),
+     from the specified location (ARG=<path>),
+     or disable it (ARG=no)
+     @<:@ARG=yes@:>@ ])],
+    [
+     AS_CASE([$withval],
+       [no],[want_boost="no";_AX_BOOST_BASE_boost_path=""],
+       [yes],[want_boost="yes";_AX_BOOST_BASE_boost_path=""],
+       [want_boost="yes";_AX_BOOST_BASE_boost_path="$withval"])
+    ],
+    [want_boost="yes"])
+
+
+AC_ARG_WITH([boost-libdir],
+  [AS_HELP_STRING([--with-boost-libdir=LIB_DIR],
+    [Force given directory for boost libraries.
+     Note that this will override library path detection,
+     so use this parameter only if default library detection fails
+     and you know exactly where your boost libraries are located.])],
+  [
+   AS_IF([test -d "$withval"],
+         [_AX_BOOST_BASE_boost_lib_path="$withval"],
+    [AC_MSG_ERROR([--with-boost-libdir expected directory name])])
+  ],
+  [_AX_BOOST_BASE_boost_lib_path=""])
+
+BOOST_LDFLAGS=""
+BOOST_CPPFLAGS=""
+AS_IF([test "x$want_boost" = "xyes"],
+      [_AX_BOOST_BASE_RUNDETECT([$1],[$2],[$3])])
+AC_SUBST(BOOST_CPPFLAGS)
+AC_SUBST(BOOST_LDFLAGS)
+])
+
+
+# convert a version string in $2 to numeric and affect to polymorphic var $1
+AC_DEFUN([_AX_BOOST_BASE_TONUMERICVERSION],[
+  AS_IF([test "x$2" = "x"],[_AX_BOOST_BASE_TONUMERICVERSION_req="1.20.0"],[_AX_BOOST_BASE_TONUMERICVERSION_req="$2"])
+  _AX_BOOST_BASE_TONUMERICVERSION_req_shorten=`expr $_AX_BOOST_BASE_TONUMERICVERSION_req : '\([[0-9]]*\.[[0-9]]*\)'`
+  _AX_BOOST_BASE_TONUMERICVERSION_req_major=`expr $_AX_BOOST_BASE_TONUMERICVERSION_req : '\([[0-9]]*\)'`
+  AS_IF([test "x$_AX_BOOST_BASE_TONUMERICVERSION_req_major" = "x"],
+        [AC_MSG_ERROR([You should at least specify libboost major version])])
+  _AX_BOOST_BASE_TONUMERICVERSION_req_minor=`expr $_AX_BOOST_BASE_TONUMERICVERSION_req : '[[0-9]]*\.\([[0-9]]*\)'`
+  AS_IF([test "x$_AX_BOOST_BASE_TONUMERICVERSION_req_minor" = "x"],
+        [_AX_BOOST_BASE_TONUMERICVERSION_req_minor="0"])
+  _AX_BOOST_BASE_TONUMERICVERSION_req_sub_minor=`expr $_AX_BOOST_BASE_TONUMERICVERSION_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'`
+  AS_IF([test "X$_AX_BOOST_BASE_TONUMERICVERSION_req_sub_minor" = "X"],
+        [_AX_BOOST_BASE_TONUMERICVERSION_req_sub_minor="0"])
+  _AX_BOOST_BASE_TONUMERICVERSION_RET=`expr $_AX_BOOST_BASE_TONUMERICVERSION_req_major \* 100000 \+  $_AX_BOOST_BASE_TONUMERICVERSION_req_minor \* 100 \+ $_AX_BOOST_BASE_TONUMERICVERSION_req_sub_minor`
+  AS_VAR_SET($1,$_AX_BOOST_BASE_TONUMERICVERSION_RET)
+])
+
+dnl Run the detection of boost should be run only if $want_boost
+AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[
+    _AX_BOOST_BASE_TONUMERICVERSION(WANT_BOOST_VERSION,[$1])
+    succeeded=no
+
+
+    AC_REQUIRE([AC_CANONICAL_HOST])
+    dnl On 64-bit systems check for system libraries in both lib64 and lib.
+    dnl The former is specified by FHS, but e.g. Debian does not adhere to
+    dnl this (as it rises problems for generic multi-arch support).
+    dnl The last entry in the list is chosen by default when no libraries
+    dnl are found, e.g. when only header-only libraries are installed!
+    AS_CASE([${host_cpu}],
+      [x86_64],[libsubdirs="lib64 libx32 lib lib64"],
+      [ppc64|s390x|sparc64|aarch64|ppc64le|riscv64],[libsubdirs="lib64 lib lib64"],
+      [libsubdirs="lib"]
+    )
+
+    dnl allow for real multi-arch paths e.g. /usr/lib/x86_64-linux-gnu. Give
+    dnl them priority over the other paths since, if libs are found there, they
+    dnl are almost assuredly the ones desired.
+    AS_CASE([${host_cpu}],
+      [i?86],[multiarch_libsubdir="lib/i386-${host_os}"],
+      [multiarch_libsubdir="lib/${host_cpu}-${host_os}"]
+    )
+
+    dnl first we check the system location for boost libraries
+    dnl this location ist chosen if boost libraries are installed with the --layout=system option
+    dnl or if you install boost with RPM
+    AS_IF([test "x$_AX_BOOST_BASE_boost_path" != "x"],[
+        AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) includes in "$_AX_BOOST_BASE_boost_path/include"])
+         AS_IF([test -d "$_AX_BOOST_BASE_boost_path/include" && test -r "$_AX_BOOST_BASE_boost_path/include"],[
+           AC_MSG_RESULT([yes])
+           BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path/include"
+           for _AX_BOOST_BASE_boost_path_tmp in $multiarch_libsubdir $libsubdirs; do
+                AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) lib path in "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp"])
+                AS_IF([test -d "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" && test -r "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" ],[
+                        AC_MSG_RESULT([yes])
+                        BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp";
+                        break;
+                ],
+      [AC_MSG_RESULT([no])])
+           done],[
+      AC_MSG_RESULT([no])])
+    ],[
+        if test X"$cross_compiling" = Xyes; then
+            search_libsubdirs=$multiarch_libsubdir
+        else
+            search_libsubdirs="$multiarch_libsubdir $libsubdirs"
+        fi
+        for _AX_BOOST_BASE_boost_path_tmp in /usr /usr/local /opt /opt/local ; do
+            if test -d "$_AX_BOOST_BASE_boost_path_tmp/include/boost" && test -r "$_AX_BOOST_BASE_boost_path_tmp/include/boost" ; then
+                for libsubdir in $search_libsubdirs ; do
+                    if ls "$_AX_BOOST_BASE_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi
+                done
+                BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path_tmp/$libsubdir"
+                BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path_tmp/include"
+                break;
+            fi
+        done
+    ])
+
+    dnl overwrite ld flags if we have required special directory with
+    dnl --with-boost-libdir parameter
+    AS_IF([test "x$_AX_BOOST_BASE_boost_lib_path" != "x"],
+          [BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_lib_path"])
+
+    AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION)])
+    CPPFLAGS_SAVED="$CPPFLAGS"
+    CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+    export CPPFLAGS
+
+    LDFLAGS_SAVED="$LDFLAGS"
+    LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+    export LDFLAGS
+
+    AC_REQUIRE([AC_PROG_CXX])
+    AC_LANG_PUSH(C++)
+        AC_COMPILE_IFELSE([_AX_BOOST_BASE_PROGRAM($WANT_BOOST_VERSION)],[
+        AC_MSG_RESULT(yes)
+    succeeded=yes
+    found_system=yes
+        ],[
+        ])
+    AC_LANG_POP([C++])
+
+
+
+    dnl if we found no boost with system layout we search for boost libraries
+    dnl built and installed without the --layout=system option or for a staged(not installed) version
+    if test "x$succeeded" != "xyes" ; then
+        CPPFLAGS="$CPPFLAGS_SAVED"
+        LDFLAGS="$LDFLAGS_SAVED"
+        BOOST_CPPFLAGS=
+        if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then
+            BOOST_LDFLAGS=
+        fi
+        _version=0
+        if test -n "$_AX_BOOST_BASE_boost_path" ; then
+            if test -d "$_AX_BOOST_BASE_boost_path" && test -r "$_AX_BOOST_BASE_boost_path"; then
+                for i in `ls -d $_AX_BOOST_BASE_boost_path/include/boost-* 2>/dev/null`; do
+                    _version_tmp=`echo $i | sed "s#$_AX_BOOST_BASE_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
+                    V_CHECK=`expr $_version_tmp \> $_version`
+                    if test "x$V_CHECK" = "x1" ; then
+                        _version=$_version_tmp
+                    fi
+                    VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
+                    BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path/include/boost-$VERSION_UNDERSCORE"
+                done
+                dnl if nothing found search for layout used in Windows distributions
+                if test -z "$BOOST_CPPFLAGS"; then
+                    if test -d "$_AX_BOOST_BASE_boost_path/boost" && test -r "$_AX_BOOST_BASE_boost_path/boost"; then
+                        BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path"
+                    fi
+                fi
+                dnl if we found something and BOOST_LDFLAGS was unset before
+                dnl (because "$_AX_BOOST_BASE_boost_lib_path" = ""), set it here.
+                if test -n "$BOOST_CPPFLAGS" && test -z "$BOOST_LDFLAGS"; then
+                    for libsubdir in $libsubdirs ; do
+                        if ls "$_AX_BOOST_BASE_boost_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi
+                    done
+                    BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$libsubdir"
+                fi
+            fi
+        else
+            if test "x$cross_compiling" != "xyes" ; then
+                for _AX_BOOST_BASE_boost_path in /usr /usr/local /opt /opt/local ; do
+                    if test -d "$_AX_BOOST_BASE_boost_path" && test -r "$_AX_BOOST_BASE_boost_path" ; then
+                        for i in `ls -d $_AX_BOOST_BASE_boost_path/include/boost-* 2>/dev/null`; do
+                            _version_tmp=`echo $i | sed "s#$_AX_BOOST_BASE_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
+                            V_CHECK=`expr $_version_tmp \> $_version`
+                            if test "x$V_CHECK" = "x1" ; then
+                                _version=$_version_tmp
+                                best_path=$_AX_BOOST_BASE_boost_path
+                            fi
+                        done
+                    fi
+                done
+
+                VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
+                BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE"
+                if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then
+                    for libsubdir in $libsubdirs ; do
+                        if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi
+                    done