Commit df4a02be authored by Neil McGlohon's avatar Neil McGlohon

Merge branch 'dfp-online-workloads' into 'master'

# Conflicts:
#   src/network-workloads/model-net-mpi-replay.c
#   src/networks/model-net/dragonfly-custom.C
parents e481d3ec 4bf55ecf
......@@ -13,6 +13,8 @@ EXTRA_PROGRAMS =
CLEANFILES = $(bin_SCRIPTS)
EXTRA_DIST =
BUILT_SOURCES =
AM_LDFLAGS =
# pkgconfig files
pkgconfigdir = $(libdir)/pkgconfig
......@@ -57,6 +59,12 @@ AM_CPPFLAGS += ${RECORDER_CPPFLAGS}
src_libcodes_la_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
endif
if USE_ONLINE
AM_CPPFLAGS += ${ARGOBOTS_CFLAGS} ${SWM_CFLAGS} -DUSE_ONLINE=1
LDADD += ${SWM_LIBS} ${ARGOBOTS_LIBS}
src_libcodes_la_SOURCES += src/workload/methods/codes-online-comm-wrkld.C
endif
if USE_DUMPI
AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
src_libcodes_la_SOURCES += src/workload/methods/codes-dumpi-trace-nw-wrkld.c
......
......@@ -19,6 +19,9 @@ extern "C" {
#include <ross.h>
#include "configuration.h"
#ifdef USE_ONLINE
#include <abt.h>
#endif
#define MAX_NAME_LENGTH_WKLD 512
/* implementations included with codes */
......@@ -30,6 +33,7 @@ typedef struct recorder_params recorder_params;
/* struct to hold the actual data from a single MPI event*/
typedef struct dumpi_trace_params dumpi_trace_params;
typedef struct checkpoint_wrkld_params checkpoint_wrkld_params;
typedef struct online_comm_params online_comm_params;
struct iomock_params
{
......@@ -77,6 +81,11 @@ struct dumpi_trace_params {
#endif
};
struct online_comm_params {
char workload_name[MAX_NAME_LENGTH_WKLD];
char file_path[MAX_NAME_LENGTH_WKLD];
int nprocs;
};
struct checkpoint_wrkld_params
{
int nprocs; /* number of workload processes */
......@@ -218,14 +227,14 @@ struct codes_workload_op
} collective;
struct {
int count;
unsigned int* req_ids;
uint32_t* req_ids;
} waits;
struct {
unsigned int req_id;
uint32_t req_id;
} wait;
struct
{
unsigned int req_id;
uint32_t req_id;
}
free;
}u;
......@@ -306,6 +315,13 @@ int codes_workload_get_rank_cnt(
const char* params,
int app_id);
/* Finalize the workload */
int codes_workload_finalize(
const char* type,
const char* params,
int app_id,
int rank);
/* for debugging/logging: print an individual operation to the specified file */
void codes_workload_print_op(
FILE *f,
......@@ -324,6 +340,7 @@ struct codes_workload_method
void (*codes_workload_get_next)(int app_id, int rank, struct codes_workload_op *op);
void (*codes_workload_get_next_rc2)(int app_id, int rank);
int (*codes_workload_get_rank_cnt)(const char* params, int app_id);
int (*codes_workload_finalize)(const char* params, int app_id, int rank);
};
......
......@@ -9,6 +9,7 @@
*/
#include <map>
#include <vector>
#include <set>
#include "codes/codes.h"
#include "codes/model-net.h"
......@@ -23,8 +24,8 @@ using namespace std;
enum ConnectionType
{
CONN_LOCAL = 1,
CONN_GLOBAL,
CONN_TERMINAL
CONN_GLOBAL = 2,
CONN_TERMINAL = 3
};
/**
......@@ -42,6 +43,11 @@ struct Connection
ConnectionType conn_type; //type of the connection: CONN_LOCAL, CONN_GLOBAL, or CONN_TERMINAL
};
inline bool operator<(const Connection& lhs, const Connection& rhs)
{
return lhs.port < rhs.port;
}
/**
* @class ConnectionManager
*
......@@ -69,6 +75,12 @@ class ConnectionManager {
map< int, Connection > _portMap; //Mapper for ports to connections
vector< int > _other_groups_i_connect_to;
set< int > _other_groups_i_connect_to_set;
map< int, vector< Connection > > _connections_to_groups_map; //maps group ID to connections to said group
map< int, vector< Connection > > _all_conns_by_type_map;
// map< int, vector< Connection > > intermediateRouterToGroupMap; //maps group id to list of routers that connect to it.
// //ex: intermediateRouterToGroupMap[3] returns a vector
// //of connections from this router to routers that have
......@@ -191,6 +203,17 @@ public:
*/
vector< Connection > get_connections_by_type(ConnectionType type);
/**
* @brief returns a vector of all group IDs that the router has a global connection to
* @note this does not include the router's own group as that is a given
*/
vector< int > get_connected_group_ids();
/**
*
*/
void solidify_connections();
/**
* @brief prints out the state of the connection manager
*/
......@@ -255,6 +278,9 @@ void ConnectionManager::add_connection(int dest_gid, ConnectionType type)
// TW_ERROR(TW_LOC, "add_connection(dest_id, type): Undefined connection type\n");
}
if(conn.dest_group_id != conn.src_group_id)
_other_groups_i_connect_to_set.insert(conn.dest_group_id);
_portMap[conn.port] = conn;
}
......@@ -393,46 +419,90 @@ vector< Connection > ConnectionManager::get_connections_to_gid(int dest_gid, Con
vector< Connection > ConnectionManager::get_connections_to_group(int dest_group_id)
{
vector< Connection > conns_to_group;
return _connections_to_groups_map[dest_group_id];
}
map< int, vector< Connection > >::iterator it = globalConnections.begin();
for(; it != globalConnections.end(); it++) //iterate over each router that is connected to source
{
vector< Connection >::iterator conns_to_router;
for(conns_to_router = (it->second).begin(); conns_to_router != (it->second).end(); conns_to_router++) //iterate over each connection to a specific router
vector< Connection > ConnectionManager::get_connections_by_type(ConnectionType type)
{
switch (type)
{
if ((*conns_to_router).dest_group_id == dest_group_id) {
conns_to_group.push_back(*conns_to_router);
}
case CONN_LOCAL:
return _all_conns_by_type_map[CONN_LOCAL];
break;
case CONN_GLOBAL:
return _all_conns_by_type_map[CONN_GLOBAL];
break;
case CONN_TERMINAL:
return _all_conns_by_type_map[CONN_TERMINAL];
break;
default:
tw_error(TW_LOC, "Bad enum type\n");
}
}
return conns_to_group;
}
vector< Connection > ConnectionManager::get_connections_by_type(ConnectionType type)
vector< int > ConnectionManager::get_connected_group_ids()
{
map< int, vector< Connection > > theMap;
switch (type)
return _other_groups_i_connect_to;
}
void ConnectionManager::solidify_connections()
{
//-- other groups connect to
set< int >::iterator it;
for(it = _other_groups_i_connect_to_set.begin(); it != _other_groups_i_connect_to_set.end(); it++)
{
case CONN_LOCAL:
theMap = intraGroupConnections;
break;
case CONN_GLOBAL:
theMap = globalConnections;
break;
case CONN_TERMINAL:
theMap = terminalConnections;
break;
_other_groups_i_connect_to.push_back(*it);
}
vector< Connection > retVec;
map< int, vector< Connection > >::iterator it;
for(it = theMap.begin(); it != theMap.end(); it++)
//--connections to group
for(it = _other_groups_i_connect_to_set.begin(); it != _other_groups_i_connect_to_set.end(); it++)
{
retVec.insert(retVec.end(), (*it).second.begin(), (*it).second.end());
int dest_group_id = *it;
vector< Connection > conns_to_group;
map< int, vector< Connection > >::iterator itg = globalConnections.begin();
for(; itg != globalConnections.end(); itg++) //iterate over each router that is connected to source
{
vector< Connection >::iterator conns_to_router;
for(conns_to_router = (itg->second).begin(); conns_to_router != (itg->second).end(); conns_to_router++) //iterate over each connection to a specific router
{
if ((*conns_to_router).dest_group_id == dest_group_id) {
conns_to_group.push_back(*conns_to_router);
}
}
}
_connections_to_groups_map[dest_group_id] = conns_to_group;
}
return retVec;
//--get connections by type
map< int, vector< Connection > > theMap;
for ( int enum_int = CONN_LOCAL; enum_int != CONN_TERMINAL + 1; enum_int++ )
{
switch (enum_int)
{
case CONN_LOCAL:
theMap = intraGroupConnections;
break;
case CONN_GLOBAL:
theMap = globalConnections;
break;
case CONN_TERMINAL:
theMap = terminalConnections;
break;
default:
tw_error(TW_LOC, "Bad enum type\n");
}
vector< Connection > retVec;
map< int, vector< Connection > >::iterator it;
for(it = theMap.begin(); it != theMap.end(); it++)
{
retVec.insert(retVec.end(), (*it).second.begin(), (*it).second.end());
}
_all_conns_by_type_map[enum_int] = retVec;
}
}
......
......@@ -5,6 +5,10 @@ AC_PREREQ([2.67])
AC_INIT([codes], [0.6.0], [http://trac.mcs.anl.gov/projects/codes/newticket],[],[http://www.mcs.anl.gov/projects/codes/])
LT_INIT
#WRAP SOME OPTION AROUND THIS - IT'S NOT REQUIRED FOR 99% OF CODES
AX_BOOST_BASE([1.66])
AC_CANONICAL_TARGET
AC_CANONICAL_SYSTEM
AC_CANONICAL_HOST
......@@ -26,6 +30,7 @@ AC_PROG_CC
AM_PROG_CC_C_O
AC_PROG_CXX
AC_PROG_CXXCPP
AX_CXX_COMPILE_STDCXX(11, noext, mandatory)
AC_PROG_RANLIB
PKG_PROG_PKG_CONFIG
......@@ -104,6 +109,24 @@ fi
AM_CONDITIONAL(USE_DARSHAN, [test "x${use_darshan}" = xyes])
# check for Argobots
AC_ARG_WITH([online],[AS_HELP_STRING([--with-online@<:@=DIR@:>@],
[Build with the online workloads and argobots support])],
[use_online=yes],[use_online=no])
if test "x${use_online}" != "x" ; then
AM_CONDITIONAL(USE_ONLINE, true)
PKG_CHECK_MODULES_STATIC([ARGOBOTS], [argobots], [],
[AC_MSG_ERROR([Could not find working argobots installation via pkg-config])])
PKG_CHECK_MODULES_STATIC([SWM], [swm], [],
[AC_MSG_ERROR([Could not find working swm installation via pkg-config])])
PKG_CHECK_VAR([SWM_DATAROOTDIR], [swm], [datarootdir], [],
[AC_MSG_ERROR[Could not find shared directory in SWM]])
AC_DEFINE_UNQUOTED([SWM_DATAROOTDIR], ["$SWM_DATAROOTDIR"], [if using json
data files])
else
AM_CONDITIONAL(USE_ONLINE, false)
fi
# check for Recorder
AM_CONDITIONAL(USE_RECORDER, true)
RECORDER_CPPFLAGS="-DUSE_RECORDER=1"
......
# ===========================================================================
# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional])
#
# DESCRIPTION
#
# Check for baseline language coverage in the compiler for the specified
# version of the C++ standard. If necessary, add switches to CXX and
# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard)
# or '14' (for the C++14 standard).
#
# The second argument, if specified, indicates whether you insist on an
# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g.
# -std=c++11). If neither is specified, you get whatever works, with
# preference for an extended mode.
#
# The third argument, if specified 'mandatory' or if left unspecified,
# indicates that baseline support for the specified C++ standard is
# required and that the macro should error out if no mode with that
# support is found. If specified 'optional', then configuration proceeds
# regardless, after defining HAVE_CXX${VERSION} if and only if a
# supporting mode is found.
#
# LICENSE
#
# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com>
# Copyright (c) 2012 Zack Weinberg <zackw@panix.com>
# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu>
# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com>
# Copyright (c) 2015 Paul Norman <penorman@mac.com>
# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved. This file is offered as-is, without any
# warranty.
#serial 9
dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro
dnl (serial version number 13).
AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"],
[$1], [14], [ax_cxx_compile_alternatives="14 1y"],
[$1], [17], [ax_cxx_compile_alternatives="17 1z"],
[m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl
m4_if([$2], [], [],
[$2], [ext], [],
[$2], [noext], [],
[m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl
m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true],
[$3], [mandatory], [ax_cxx_compile_cxx$1_required=true],
[$3], [optional], [ax_cxx_compile_cxx$1_required=false],
[m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])])
AC_LANG_PUSH([C++])dnl
ac_success=no
m4_if([$2], [noext], [], [dnl
if test x$ac_success = xno; then
for alternative in ${ax_cxx_compile_alternatives}; do
switch="-std=gnu++${alternative}"
cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
$cachevar,
[ac_save_CXX="$CXX"
CXX="$CXX $switch"
AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
[eval $cachevar=yes],
[eval $cachevar=no])
CXX="$ac_save_CXX"])
if eval test x\$$cachevar = xyes; then
CXX="$CXX $switch"
if test -n "$CXXCPP" ; then
CXXCPP="$CXXCPP $switch"
fi
ac_success=yes
break
fi
done
fi])
m4_if([$2], [ext], [], [dnl
if test x$ac_success = xno; then
dnl HP's aCC needs +std=c++11 according to:
dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf
dnl Cray's crayCC needs "-h std=c++11"
for alternative in ${ax_cxx_compile_alternatives}; do
for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do
cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
$cachevar,
[ac_save_CXX="$CXX"
CXX="$CXX $switch"
AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
[eval $cachevar=yes],
[eval $cachevar=no])
CXX="$ac_save_CXX"])
if eval test x\$$cachevar = xyes; then
CXX="$CXX $switch"
if test -n "$CXXCPP" ; then
CXXCPP="$CXXCPP $switch"
fi
ac_success=yes
break
fi
done
if test x$ac_success = xyes; then
break
fi
done
fi])
AC_LANG_POP([C++])
if test x$ax_cxx_compile_cxx$1_required = xtrue; then
if test x$ac_success = xno; then
AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.])
fi
fi
if test x$ac_success = xno; then
HAVE_CXX$1=0
AC_MSG_NOTICE([No compiler with C++$1 support was found])
else
HAVE_CXX$1=1
AC_DEFINE(HAVE_CXX$1,1,
[define if the compiler supports basic C++$1 syntax])
fi
AC_SUBST(HAVE_CXX$1)
])
dnl Test body for checking C++11 support
m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11],
_AX_CXX_COMPILE_STDCXX_testbody_new_in_11
)
dnl Test body for checking C++14 support
m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
_AX_CXX_COMPILE_STDCXX_testbody_new_in_11
_AX_CXX_COMPILE_STDCXX_testbody_new_in_14
)
m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17],
_AX_CXX_COMPILE_STDCXX_testbody_new_in_11
_AX_CXX_COMPILE_STDCXX_testbody_new_in_14
_AX_CXX_COMPILE_STDCXX_testbody_new_in_17
)
dnl Tests for new features in C++11
m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[
// If the compiler admits that it is not ready for C++11, why torture it?
// Hopefully, this will speed up the test.
#ifndef __cplusplus
#error "This is not a C++ compiler"
#elif __cplusplus < 201103L
#error "This is not a C++11 compiler"
#else
namespace cxx11
{
namespace test_static_assert
{
template <typename T>
struct check
{
static_assert(sizeof(int) <= sizeof(T), "not big enough");
};
}
namespace test_final_override
{
struct Base
{
virtual void f() {}
};
struct Derived : public Base
{
virtual void f() override {}
};
}
namespace test_double_right_angle_brackets
{
template < typename T >
struct check {};
typedef check<void> single_type;
typedef check<check<void>> double_type;
typedef check<check<check<void>>> triple_type;
typedef check<check<check<check<void>>>> quadruple_type;
}
namespace test_decltype
{
int
f()
{
int a = 1;
decltype(a) b = 2;
return a + b;
}
}
namespace test_type_deduction
{
template < typename T1, typename T2 >
struct is_same
{
static const bool value = false;
};
template < typename T >
struct is_same<T, T>
{
static const bool value = true;
};
template < typename T1, typename T2 >
auto
add(T1 a1, T2 a2) -> decltype(a1 + a2)
{
return a1 + a2;
}
int
test(const int c, volatile int v)
{
static_assert(is_same<int, decltype(0)>::value == true, "");
static_assert(is_same<int, decltype(c)>::value == false, "");
static_assert(is_same<int, decltype(v)>::value == false, "");
auto ac = c;
auto av = v;
auto sumi = ac + av + 'x';
auto sumf = ac + av + 1.0;
static_assert(is_same<int, decltype(ac)>::value == true, "");
static_assert(is_same<int, decltype(av)>::value == true, "");
static_assert(is_same<int, decltype(sumi)>::value == true, "");
static_assert(is_same<int, decltype(sumf)>::value == false, "");
static_assert(is_same<int, decltype(add(c, v))>::value == true, "");
return (sumf > 0.0) ? sumi : add(c, v);
}
}
namespace test_noexcept
{
int f() { return 0; }
int g() noexcept { return 0; }
static_assert(noexcept(f()) == false, "");
static_assert(noexcept(g()) == true, "");
}
namespace test_constexpr
{
template < typename CharT >
unsigned long constexpr
strlen_c_r(const CharT *const s, const unsigned long acc) noexcept
{
return *s ? strlen_c_r(s + 1, acc + 1) : acc;
}
template < typename CharT >
unsigned long constexpr
strlen_c(const CharT *const s) noexcept
{
return strlen_c_r(s, 0UL);
}
static_assert(strlen_c("") == 0UL, "");
static_assert(strlen_c("1") == 1UL, "");
static_assert(strlen_c("example") == 7UL, "");
static_assert(strlen_c("another\0example") == 7UL, "");
}
namespace test_rvalue_references
{
template < int N >
struct answer
{
static constexpr int value = N;
};
answer<1> f(int&) { return answer<1>(); }
answer<2> f(const int&) { return answer<2>(); }
answer<3> f(int&&) { return answer<3>(); }
void
test()
{
int i = 0;
const int c = 0;
static_assert(decltype(f(i))::value == 1, "");
static_assert(decltype(f(c))::value == 2, "");
static_assert(decltype(f(0))::value == 3, "");
}
}
namespace test_uniform_initialization
{
struct test
{
static const int zero {};
static const int one {1};
};
static_assert(test::zero == 0, "");
static_assert(test::one == 1, "");
}
namespace test_lambdas
{
void
test1()
{
auto lambda1 = [](){};
auto lambda2 = lambda1;
lambda1();
lambda2();
}
int
test2()
{
auto a = [](int i, int j){ return i + j; }(1, 2);
auto b = []() -> int { return '0'; }();
auto c = [=](){ return a + b; }();
auto d = [&](){ return c; }();
auto e = [a, &b](int x) mutable {
const auto identity = [](int y){ return y; };
for (auto i = 0; i < a; ++i)
a += b--;
return x + identity(a + b);
}(0);
return a + b + c + d + e;
}
int
test3()
{
const auto nullary = [](){ return 0; };
const auto unary = [](int x){ return x; };
using nullary_t = decltype(nullary);
using unary_t = decltype(unary);
const auto higher1st = [](nullary_t f){ return f(); };
const auto higher2nd = [unary](nullary_t f1){
return [unary, f1](unary_t f2){ return f2(unary(f1())); };
};
return higher1st(nullary) + higher2nd(nullary)(unary);
}
}
namespace test_variadic_templates
{
template <int...>
struct sum;
template <int N0, int... N1toN>
struct sum<N0, N1toN...>
{
static constexpr auto value = N0 + sum<N1toN...>::value;
};
template <>
struct sum<>
{
static constexpr auto value = 0;
};
static_assert(sum<>::value == 0, "");
static_assert(sum<1>::value == 1, "");
static_assert(sum<23>::value == 23, "");
static_assert(sum<1, 2>::value == 3, "");
static_assert(sum<5, 5, 11>::value == 21, "");
static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, "");
}
// http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae
// Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function
// because of this.
namespace test_template_alias_sfinae
{