...
 
Commits (10)
......@@ -61,9 +61,23 @@ endif
if USE_ONLINE
AM_CPPFLAGS += ${ARGOBOTS_CFLAGS} ${SWM_CFLAGS} -DUSE_ONLINE=1
LDADD += ${SWM_LIBS} ${ARGOBOTS_LIBS}
LDADD += ${ARGOBOTS_LIBS}
if USE_SWM
AM_CPPFLAGS += -DUSE_SWM=1
LDADD += ${SWM_LIBS}
src_libcodes_la_SOURCES += src/workload/methods/codes-online-comm-wrkld.C
endif
if USE_CONC
src_libcodes_la_SOURCES += src/workload/methods/codes-conc-online-comm-wrkld.C
AM_CPPFLAGS += ${CONCEPTUAL_CFLAGS} -DUSE_CONC=1
LDADD += ${CONCEPTUAL_LIBS}
src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-bisect.c
src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-cosmoflow.c
src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-hotpotato.c
src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-latencyall.c
src_libcodes_la_SOURCES += src/workload/conceputal-skeleton-apps/conc-latency.c
endif
endif
if USE_DUMPI
AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1
......
/*
* Copyright (C) 2017 University of Chicago.
* See COPYRIGHT notice in top-level directory.
*
*/
#ifndef CODES_CONC_ADDON_H
#define CODES_CONC_ADDON_H
#ifdef __cplusplus
extern "C" {
#endif
#ifdef USE_CONC
#include <ncptl/ncptl.h>
#endif
#include <mpi.h>
#define MAX_CONC_ARGV 128
typedef struct conc_bench_param conc_bench_param;
struct conc_bench_param {
char conc_program[MAX_CONC_ARGV];
int conc_argc;
char config_in[MAX_CONC_ARGV][MAX_CONC_ARGV];
char *conc_argv[MAX_CONC_ARGV];
};
int codes_conc_bench_load(
const char* program,
int argc,
char *argv[]);
void CODES_MPI_Comm_size (MPI_Comm comm, int *size);
void CODES_MPI_Comm_rank( MPI_Comm comm, int *rank );
void CODES_MPI_Finalize();
void CODES_MPI_Send(const void *buf,
int count,
MPI_Datatype datatype,
int dest,
int tag,
MPI_Comm comm);
void CODES_MPI_Recv(void *buf,
int count,
MPI_Datatype datatype,
int source,
int tag,
MPI_Comm comm,
MPI_Status *status);
void CODES_MPI_Sendrecv(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
int dest,
int sendtag,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
int source,
int recvtag,
MPI_Comm comm,
MPI_Status *status);
void CODES_MPI_Barrier(MPI_Comm comm);
void CODES_MPI_Isend(const void *buf,
int count,
MPI_Datatype datatype,
int dest,
int tag,
MPI_Comm comm,
MPI_Request *request);
void CODES_MPI_Irecv(void *buf,
int count,
MPI_Datatype datatype,
int source,
int tag,
MPI_Comm comm,
MPI_Request *request);
void CODES_MPI_Waitall(int count,
MPI_Request array_of_requests[],
MPI_Status array_of_statuses[]);
void CODES_MPI_Reduce(const void *sendbuf,
void *recvbuf,
int count,
MPI_Datatype datatype,
MPI_Op op,
int root,
MPI_Comm comm);
void CODES_MPI_Allreduce(const void *sendbuf,
void *recvbuf,
int count,
MPI_Datatype datatype,
MPI_Op op,
MPI_Comm comm);
void CODES_MPI_Bcast(void *buffer,
int count,
MPI_Datatype datatype,
int root,
MPI_Comm comm);
void CODES_MPI_Alltoall(const void *sendbuf,
int sendcount,
MPI_Datatype sendtype,
void *recvbuf,
int recvcount,
MPI_Datatype recvtype,
MPI_Comm comm);
void CODES_MPI_Alltoallv(const void *sendbuf,
const int *sendcounts,
const int *sdispls,
MPI_Datatype sendtype,
void *recvbuf,
const int *recvcounts,
const int *rdispls,
MPI_Datatype recvtype,
MPI_Comm comm);
/* implementation structure */
struct codes_conceptual_bench {
char *program_name; /* name of the conceptual program */
int (*conceptual_main)(int argc, char *argv[]);
};
void codes_conceptual_add_bench(struct codes_conceptual_bench const * method);
#ifdef __cplusplus
}
#endif
#endif /* CODES_CONC_ADDON_H */
......@@ -121,22 +121,48 @@ AM_CONDITIONAL(USE_DARSHAN, [test "x${use_darshan}" = xyes])
# check for Argobots
AC_ARG_WITH([online],[AS_HELP_STRING([--with-online@<:@=DIR@:>@],
[Build with the online workloads and argobots support])],
[use_online=yes],[use_online=no])
if test "x${use_online}" != "xno" ; then
[Build with the online workloads and argobots support])])
if test "x${with_online}" != "x" ; then
AM_CONDITIONAL(USE_ONLINE, true)
PKG_CHECK_MODULES_STATIC([ARGOBOTS], [argobots], [],
[AC_MSG_ERROR([Could not find working argobots installation via pkg-config])])
AC_DEFINE_UNQUOTED([ONLINE_CONFIGDIR], ["$with_online"], [if using json data files,
specify config directory])
else
AM_CONDITIONAL(USE_ONLINE, false)
fi
#check for SWM
AC_ARG_WITH([swm],[AS_HELP_STRING([--with-swm@<:@=DIR@:>@],
[location of SWM installation])])
if test "x${with_swm}" != "x" ; then
AM_CONDITIONAL(USE_SWM, true)
PKG_CHECK_MODULES_STATIC([SWM], [swm], [],
[AC_MSG_ERROR([Could not find working swm installation via pkg-config])])
PKG_CHECK_VAR([SWM_DATAROOTDIR], [swm], [datarootdir], [],
[AC_MSG_ERROR[Could not find shared directory in SWM]])
AC_DEFINE_UNQUOTED([SWM_DATAROOTDIR], ["$SWM_DATAROOTDIR"], [if using json
data files])
data files])
else
AM_CONDITIONAL(USE_ONLINE, false)
AM_CONDITIONAL(USE_SWM, false)
fi
#check for Conceptual
AC_ARG_WITH([conceptual],[AS_HELP_STRING([--with-conceptual@<:@=DIR@:>@],
[location of Conceptual installation])])
if test "x${with_conceptual}" != "x" ; then
AC_CHECK_FILES([${with_conceptual}/lib/libncptl.a],
AM_CONDITIONAL(USE_CONC, true),
AC_MSG_ERROR(Could not find Conceptual libraries libncptl.a))
CONCEPTUAL_CFLAGS="-I${with_conceptual}/include"
CONCEPTUAL_LIBS="-L${with_conceptual}/lib/ -lncptl"
AC_SUBST(CONCEPTUAL_LIBS)
AC_SUBST(CONCEPTUAL_CFLAGS)
else
AM_CONDITIONAL(USE_CONC, false)
fi
# check for Recorder
AM_CONDITIONAL(USE_RECORDER, true)
RECORDER_CPPFLAGS="-DUSE_RECORDER=1"
......
......@@ -16,6 +16,8 @@ boost_cflags=@BOOST_CFLAGS@
boost_libs=@BOOST_LIBS@
argobots_libs=@ARGOBOTS_LIBS@
argobots_cflags=@ARGOBOTS_CFLAGS@
conceptual_libs=@CONCEPTUAL_LIBS@
conceptual_cflags=@CONCEPTUAL_CFLAGS@
swm_libs=@SWM_LIBS@
swm_cflags=@SWM_CFLAGS@
swm_datarootdir=@SWM_DATAROOTDIR@
......@@ -25,5 +27,5 @@ Description: Base functionality for CODES storage simulation
Version: @PACKAGE_VERSION@
URL: http://trac.mcs.anl.gov/projects/CODES
Requires:
Libs: -L${libdir} -lcodes ${ross_libs} ${argobots_libs} ${swm_libs} ${darshan_libs} ${dumpi_libs} ${cortex_libs}
Cflags: -I${includedir} -I${swm_datarootdir} ${ross_cflags} ${darshan_cflags} ${swm_cflags} ${argobots_cflags} ${dumpi_cflags} ${cortex_cflags}
Libs: -L${libdir} -lcodes ${ross_libs} ${argobots_libs} ${conceptual_libs} ${swm_libs} ${darshan_libs} ${dumpi_libs} ${cortex_libs}
Cflags: -I${includedir} ${swm_datarootdir} ${ross_cflags} ${darshan_cflags} ${swm_cflags} ${argobots_cflags} ${conceptual_cflags} ${dumpi_cflags} ${cortex_cflags}
/**********************************************************************
* This file was generated by coNCePTuaL on Mon Jan 7 23:10:02 2019
* using the c_mpi backend (C + MPI).
* Do not modify this file; modify /Users/xin/macworkspace/codes-dev/codes/scripts/conceptual_benchmarks/bisect.ncptl instead.
*
* Entire source program
* ---------------------
* ###################################################
* # Measure random bisection-bandwidth patterns #
* # By Scott Pakin <pakin@lanl.gov> #
* # #
* # Inspired by Hoefler, Schneider, and Lumsdaine's #
* # "Multistage Switches are not Crossbars" paper #
* ###################################################
*
* Require language version "1.5".
*
* nummsgs is "Number of messages per trial" and comes from "--nummsgs" or "-n" with default 100.
* wups is "Number of warmup messages" and comes from "--wups" or "-w" with default 3.
* msgsize is "Message size in bytes" and comes from "--bytes" or "-b" with default 1M.
* numtrials is "Number of bisection patterns" and comes from "--trials" or "-t" with default 5000.
* rounding is "Round measurements to the nearest N" and comes from "--round" or "-r" with default 5.
*
* For each trial in {1, ..., numtrials} {
* task 0 is assigned to processor 0 then
* task 0 outputs "Testing random bisection pattern " and trial and "/" and numtrials then
* all tasks are assigned to a random processor then
* tasks src such that src is even send wups msgsize-byte messages to task src+1 then
* all tasks synchronize then
* all tasks reset their counters then
* tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then
* all tasks synchronize then
* all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as "Bandwidth (MiB/s)"
* }
**********************************************************************/
/*****************
* Include files *
*****************/
/* Header files needed by all C-based backends */
#include <stdio.h>
#include <string.h>
#include <ncptl/ncptl.h>
/* Header files specific to the c_mpi backend */
#include <mpi.h>
#include <stdarg.h>
/**********
* Macros *
**********/
/* Define the maximum loop trip count that we're willing to unroll fully. */
#define CONC_MAX_UNROLL 5
/* Specify the minimum number of trial iterations in each FOR <time> loop. */
#define CONC_FOR_TIME_TRIALS 1
/* Define a macro that rounds a double to a ncptl_int. */
#define CONC_DBL2INT(D) ((ncptl_int)((D)+0.5))
/* Define a macro that increments a buffer pointer by a byte offset. */
#define CONC_GETBUFPTR(S) ((void *)((char *)thisev->s.S.buffer + thisev->s.S.bufferofs))
/* Implement ncptl_func_task_of in terms of ncptl_physical_to_virtual. */
#define ncptl_func_task_of(P) ((ncptl_int)(P) < 0LL || (ncptl_int)(P) >= var_num_tasks ? -1LL : ncptl_physical_to_virtual (procmap, (ncptl_int)(P)))
#define ncptl_dfunc_task_of(P) ((double) ncptl_func_task_of(P))
/* Implement ncptl_func_processor_of in terms of ncptl_virtual_to_physical. */
#define ncptl_func_processor_of(V) ((ncptl_int)(V) < 0LL || (ncptl_int)(V) >= var_num_tasks ? -1LL : ncptl_virtual_to_physical (procmap, (ncptl_int)(V)))
#define ncptl_dfunc_processor_of(V) ((double) ncptl_func_processor_of(V))
/* Define a macro that increments REDUCE's alternate buffer pointer by a byte offset. */
#define CONC_GETALTBUFPTR(S) ((void *)((char *)thisev->s.S.altbuffer + thisev->s.S.bufferofs))
/* Estimate the number of unique communicators that this program will need.
* (The tradeoff is one of initialization time versus memory consumption.) */
#define ESTIMATED_COMMUNICATORS 128
/* Specify an operation to use for all reduction operations. */
#define REDUCE_OPERATION MPI_SUM
#define REDUCE_OPERATION_NAME "MPI_SUM"
/*********************
* Type declarations *
*********************/
/* Enumerate the various mechanisms used to implement MULTICAST statements. */
typedef enum {
CONC_MCAST_MPI_BCAST, /* One to many */
CONC_MCAST_MPI_ALLTOALL, /* Many to many, same data to all */
CONC_MCAST_MPI_ALLTOALLV, /* General many to many */
CONC_MCAST_MPI_NUM_FUNCS /* Number of the above */
} CONC_MCAST_MPI_FUNC;
/* Define the type of event to perform. */
typedef enum {
EV_SEND, /* Synchronous send */
EV_ASEND, /* Asynchronous send */
EV_RECV, /* Synchronous receive */
EV_ARECV, /* Asynchronous receive */
EV_WAIT, /* Wait for all asynchronous sends/receives to complete */
EV_DELAY, /* Spin or sleep */
EV_TOUCH, /* Touch a region of memory */
EV_SYNC, /* Barrier synchronization */
EV_RESET, /* Reset counters */
EV_STORE, /* Store all counters' current values */
EV_RESTORE, /* Restore the previously pushed counter values */
EV_FLUSH, /* Compute aggregate functions for log-file columns */
EV_MCAST, /* Synchronous multicast */
EV_REDUCE, /* Reduction with or without a subsequent multicast */
EV_BTIME, /* Beginning of a timed loop */
EV_ETIME, /* Ending of a timed loop */
EV_REPEAT, /* Repeatedly process the next N events */
EV_SUPPRESS, /* Suppress writing to the log and standard output */
EV_NEWSTMT, /* Beginning of a new top-level statement */
EV_CODE, /* None of the above */
NUM_EVS /* Number of event types in CONC_EVENT_TYPE */
} CONC_EVENT_TYPE;
/* Describe a synchronous or asynchronous send event. */
typedef struct {
ncptl_int dest; /* Destination task */
ncptl_int size; /* Number of bytes to send */
ncptl_int alignment; /* Message alignment (in bytes) */
ncptl_int pendingsends; /* # of outstanding sends */
ncptl_int pendingrecvs; /* # of outstanding receives */
ncptl_int buffernum; /* Buffer # to send from */
ncptl_int bufferofs; /* Byte offset into the message buffer */
ncptl_int tag; /* Tag to use for selective receives */
int misaligned; /* 1=misaligned from a page; 0=align as specified */
int touching; /* 1=touch every word before sending */
int verification; /* 1=fill message buffer with known contents */
void *buffer; /* Pointer to message memory */
MPI_Request * handle; /* MPI handle representing an asynchronous send */
} CONC_SEND_EVENT;
/* Describe a synchronous or asynchronous receive event. */
typedef struct {
ncptl_int source; /* Source task */
ncptl_int size; /* Number of bytes to receive */
ncptl_int alignment; /* Message alignment (in bytes) */
ncptl_int pendingsends; /* # of outstanding sends */
ncptl_int pendingrecvs; /* # of outstanding receives */
ncptl_int buffernum; /* Buffer # to receive into */
ncptl_int bufferofs; /* Byte offset into the message buffer */
ncptl_int tag; /* Tag to use for selective receives */
int misaligned; /* 1=misaligned from a page; 0=align as specified */
int touching; /* 1=touch every word after reception */
int verification; /* 1=verify that all bits are correct */
void *buffer; /* Pointer to message memory */
MPI_Request * handle; /* MPI handle representing an asynchronous receive */
} CONC_RECV_EVENT;
/* Describe a wait-for-asynchronous-completions event. */
typedef struct {
ncptl_int numsends; /* # of sends we expect to complete. */
ncptl_int numrecvs; /* # of receives we expect to complete. */
ncptl_int numrecvbytes; /* # of bytes we expect to receive-complete */
ncptl_int *touchedlist; /* List of receives that need to be touched */
ncptl_int numtouches; /* # of elements in the above */
} CONC_WAIT_EVENT;
/* Describe a spin or sleep delay. */
typedef struct {
uint64_t microseconds; /* Length of delay in microseconds */
int spin0sleep1; /* 0=spin; 1=sleep */
} CONC_DELAY_EVENT;
/* Describe a barrier synchronization event. */
typedef struct {
MPI_Comm communicator; /* Set of tasks to synchronize */
} CONC_SYNC_EVENT;
/* Describe a walk over a memory-region. */
typedef struct {
ncptl_int regionbytes; /* Size in bytes of the region to touch */
ncptl_int bytestride; /* Stride in bytes to touch */
ncptl_int numaccesses; /* Number of words to touch */
ncptl_int wordsize; /* Size in bytes of each touch */
ncptl_int firstbyte; /* Byte offset of the first byte to touch */
} CONC_TOUCH_EVENT;
/* Describe a synchronous multicast event. */
typedef struct {
ncptl_int source; /* Source task, -1 in the many-to-many case */
ncptl_int size; /* Number of bytes to send */
ncptl_int alignment; /* Message alignment (in bytes) */
ncptl_int pendingsends; /* # of outstanding sends */
ncptl_int pendingrecvs; /* # of outstanding receives */
ncptl_int buffernum; /* Buffer # to send/receive from */
ncptl_int bufferofs; /* Byte offset into the message buffer */
ncptl_int tag; /* Tag to use for selective receives */
int misaligned; /* 1=misaligned from a page; 0=align as specified */
int touching; /* 1=touch every word before sending */
int verification; /* 1=verify that all bits are correct */
void *buffer; /* Pointer to message memory */
ncptl_int size2; /* Number of bytes to receive in the many-to-many case */
ncptl_int bufferofs2; /* Byte offset into the message buffer in the many-to-many case */
void * buffer2; /* Pointer to receive-message memory in the many-to-many case */
MPI_Comm communicator; /* Set of tasks to multicast to/from */
int root; /* source's rank within communicator */
int * sndvol; /* Volume of data to send to each rank in the communicator */
int * snddisp; /* Offset from buffer of each message to send */
int * rcvvol; /* Volume of data to receive from each rank in the communicator */
int * rcvdisp; /* Offset from buffer2 of each message to receive */
CONC_MCAST_MPI_FUNC mpi_func; /* MPI function to use to perform the multicast */
} CONC_MCAST_EVENT;
/* Describe a reduction event. */
typedef struct {
ncptl_int numitems; /* # of items to reduce */
ncptl_int itemsize; /* # of bytes per item */
ncptl_int alignment; /* Message alignment (in bytes) */
ncptl_int pendingsends; /* # of outstanding sends */
ncptl_int pendingrecvs; /* # of outstanding receives */
ncptl_int buffernum; /* Buffer # to send/receive from */
ncptl_int bufferofs; /* Byte offset into the message buffer */
ncptl_int tag; /* Tag to use for selective receives */
int misaligned; /* 1=misaligned from a page; 0=align as specified */
int touching; /* 1=touch every word before sending/after receiving */
int sending; /* 1=we're a sender */
int receiving; /* 1=we're a receiver */
void *buffer; /* Pointer to message memory */
void * altbuffer; /* Pointer to additional message memory */
MPI_Comm sendcomm; /* Set of tasks to reduce from */
MPI_Comm recvcomm; /* Set of tasks to reduce to */
MPI_Datatype datatype; /* MPI datatype to reduce */
int reducetype; /* 0=reduce; 1=allreduce; 2=reduce+bcast */
ncptl_int reduceroot; /* Root task of the reduction if reducetype is 0 or 2 */
ncptl_int bcastroot; /* Root task of the multicast if reducetype is 2 */
} CONC_REDUCE_EVENT;
/* Describe an event representing the beginning of a timed loop. */
typedef struct {
uint64_t usecs; /* Requested loop duration */
uint64_t warmup_usecs; /* Requested duration of warmup loops */
uint64_t starttime; /* Time at which the loop state last changed */
uint64_t itersleft; /* # of iterations remaining */
uint64_t previters; /* # of iterations we performed last time */
int prev_quiet; /* Previous value of suppress_output */
int timing_trial; /* 1=performing a timing trial; 0=running for real */
volatile int finished; /* 1=time has expired; 0=still ticking */
ncptl_int var_total_bytes; /* Cached copy of var_total_bytes to restore after the trial runs */
} CONC_BTIME_EVENT;
/* Describe an event representing the end of a timed loop. */
typedef struct {
ncptl_int begin_event; /* Index into eventlist[] of the corresponding BTIME event */
} CONC_ETIME_EVENT;
/* Describe an event representing repetitions of subsequent events. */
typedef struct {
ncptl_int end_event; /* Index into eventlist[] of the last event to repeat */
ncptl_int numreps; /* # of repetitions to perform */
} CONC_REPEAT_EVENT;
/* Describe an event representing output suppression (either on or off). */
typedef struct conc_suppress_event {
int quiet; /* 0=allow output; 1=suppress it */
int prev_quiet; /* Previous value of suppress_output */
ncptl_int matching_event; /* Event ID of the "suppression on" event */
ncptl_int var_total_bytes; /* Cached copy of var_total_bytes to restore after re-enabling output */
uint64_t stop_elapsed_usecs; /* Time at which we suppressed output */
} CONC_SUPPRESS_EVENT;
/* Describe an event representing arbitrary code to execute at run time. */
typedef struct {
ncptl_int number; /* Unique number corresponding to a specific piece of code */NCPTL_VIRT_PHYS_MAP *procmap; /* Current mapping between tasks and processors */
ncptl_int var_numtrials; /* Copy of var_numtrials to use within a piece of code */
ncptl_int var_trial; /* Copy of var_trial to use within a piece of code */
ncptl_int var_rounding; /* Copy of var_rounding to use within a piece of code */
} CONC_CODE_EVENT;
/* Describe an arbitrary coNCePTuaL event. */
typedef struct {
CONC_EVENT_TYPE type; /* Type of event */
union {
CONC_SEND_EVENT send; /* Send state */
CONC_RECV_EVENT recv; /* Receive state */
CONC_WAIT_EVENT wait; /* Wait-for-completions state */
CONC_DELAY_EVENT delay; /* State for spins and sleeps */
CONC_TOUCH_EVENT touch; /* State for memory touching */
CONC_SYNC_EVENT sync; /* Synchronization state */
CONC_MCAST_EVENT mcast; /* Multicast state */
CONC_REDUCE_EVENT reduce; /* Reduction state */
CONC_BTIME_EVENT btime; /* Timed-loop state */
CONC_ETIME_EVENT etime; /* Additional timed-loop state */
CONC_REPEAT_EVENT rep; /* Repeated-events state */
CONC_SUPPRESS_EVENT suppress; /* State for suppressing output */
CONC_CODE_EVENT code; /* State for arbitrary code */
} s;
} CONC_EVENT;
/* Fully specify an arbitrary for() loop (used by FOR EACH). */
typedef struct {
NCPTL_QUEUE *list_comp; /* NULL=ordinary list; other=list comprehension values */
int integral; /* 1=integral values; 0=floating-point values */
enum { /* Comparison of loop variable to end variable */
CONC_LEQ, /* Increasing progression */
CONC_GEQ /* Decreasing progression */
} comparator;
enum { /* How to increment the loop variable */
CONC_ADD, /* Arithmetically */
CONC_MULT, /* Geometrically increasing */
CONC_DIV /* Geometrically decreasing */
} increment;
union {
struct {
ncptl_int loopvar; /* Loop variable */
ncptl_int prev_loopvar; /* Previous value of loop variable */
ncptl_int startval; /* Initial value of loop variable */
ncptl_int endval; /* Value not to exceed */
ncptl_int incval; /* Loop-variable increment */
} i;
struct {
double loopvar; /* Loop variable */
double prev_loopvar; /* Previous value of loop variable */
double startval; /* Initial value of loop variable */
double endval; /* Value not to exceed */
double incval; /* Loop-variable increment */
} d;
} u;
} LOOPBOUNDS;
/********************
* Global variables *
********************/
/* Variables exported to coNCePTuaL programs */
static ncptl_int var_bytes_received = 0; /* Total number of bytes received */
static ncptl_int var_msgs_received = 0; /* Total number of messages received */
static ncptl_int var_bit_errors = 0; /* Total number of bit errors observed */
static ncptl_int var_total_msgs = 0; /* Sum of messages sent and messages received */
static ncptl_int var_msgs_sent = 0; /* Total number of messages sent */
static ncptl_int var_bytes_sent = 0; /* Total number of bytes sent */
static ncptl_int var_num_tasks = 1; /* Number of tasks running the program */
static ncptl_int var_elapsed_usecs = 0; /* Elapsed time in microseconds */
static ncptl_int var_total_bytes = 0; /* Sum of bytes sent and bytes received */
/* Dummy variable to help mark other variables as used */
union {
ncptl_int ni;
int i;
void *vp;
} conc_dummy_var;
/* Variables used internally by boilerplate code */
static uint64_t starttime; /* Time the clock was last reset (microseconds) */
static ncptl_int pendingrecvs = 0; /* Current # of outstanding receives */
static ncptl_int pendingrecvbytes = 0; /* Current # of bytes in outstanding receives */
static NCPTL_QUEUE *touchedqueue; /* Queue of asynchronous receives to touch */
static ncptl_int pendingsends = 0; /* Current # of outstanding sends */
static NCPTL_QUEUE *eventqueue; /* List of coNCePTuaL events to perform */
static int within_time_loop = 0; /* 1=we're within a FOR <time> loop */
static int suppress_output = 0; /* 1=suppress output to stdout and the log file */
static void *touch_region = NULL; /* Memory region to touch */
static ncptl_int touch_region_size = 0; /* # of bytes in the above */
static int virtrank; /* This task's virtual rank in the computation */
static int physrank; /* This task's physical rank in the computation */
static NCPTL_VIRT_PHYS_MAP *procmap; /* Virtual to physical rank mapping */
static ncptl_int random_seed; /* Seed for the random-number generator */
static NCPTL_LOG_FILE_STATE *logstate; /* Opaque object representing all log-file state */
static char *logfile_uuid; /* Execution UUID to write to every log file */
static char *logfiletmpl; /* Template for the log file's name */
static char *logfiletmpl_default; /* Default value of the above */
/* Global variables specific to the c_mpi backend */
static ncptl_int mpi_is_running = 0; /* 1=MPI has been initialized */
static NCPTL_QUEUE * recvreqQ; /* List of MPI receive requests */
static MPI_Request * recvrequests; /* List version of recvreqQ */
static NCPTL_QUEUE * recvstatQ; /* List of MPI receive statuses */
static MPI_Status * recvstatuses; /* List version of recvstatQ */
static NCPTL_QUEUE * sendreqQ; /* List of MPI send requests */
static MPI_Request * sendrequests; /* List version of sendreqQ */
static NCPTL_QUEUE * sendstatQ; /* List of MPI send statuses */
static MPI_Status * sendstatuses; /* List version of sendstatQ */
static NCPTL_SET * communicators; /* Map from an array of processor flags to an MPI communicator */
static MPI_Errhandler mpi_error_handler; /* Handle to handle_MPI_error() */
static ncptl_int mpi_tag_ub; /* Upper bound on an MPI tag value */
static ncptl_int conc_mcast_tallies[CONC_MCAST_MPI_NUM_FUNCS] = {0}; /* Tallies of (static) multicast implementation functions */
/* Program-specific variables */
ncptl_int var_nummsgs; /* Number of messages per trial (command-line argument) */
ncptl_int var_wups; /* Number of warmup messages (command-line argument) */
ncptl_int var_msgsize; /* Message size in bytes (command-line argument) */
ncptl_int var_numtrials; /* Number of bisection patterns (command-line argument) */
ncptl_int var_rounding; /* Round measurements to the nearest N (command-line argument) */
/*************************
* Function declarations *
*************************/
/* Make MPI errors invoke ncptl_fatal(). */
static void handle_MPI_error (MPI_Comm *comm, int *errcode, ...)
{
va_list args;
char errstring[MPI_MAX_ERROR_STRING];
int errstrlen;
va_start (args, errcode);
if (MPI_Error_string (*errcode, errstring, &errstrlen) == MPI_SUCCESS)
ncptl_fatal ("MPI run-time error: %s", errstring);
else
ncptl_fatal ("MPI aborted with unrecognized error code %d", *errcode);
conc_dummy_var.vp = (void *) comm; /* Prevent the compiler from complaining that comm is unused. */
va_end (args);
}
/* Perform the equivalent of MPI_Comm_rank() for an arbitrary process. */
static int rank_in_MPI_communicator (MPI_Comm subcomm, int global_rank)
{
MPI_Group world_group; /* Group associated with MPI_COMM_WORLD */
MPI_Group subgroup; /* Group associate with subcomm */
int subrank; /* global_rank's rank within subcomm */
MPI_Comm_group (MPI_COMM_WORLD, &world_group);
MPI_Comm_group (subcomm, &subgroup);
MPI_Group_translate_ranks (world_group, 1, &global_rank, subgroup, &subrank);
return subrank;
}
/* Map an arbitrary tag to within MPI's valid range of [0, mpi_tag_ub]. */
static ncptl_int map_tag_into_MPI_range (ncptl_int tag)
{
if (tag == NCPTL_INT_MIN)
/* Avoid taking the absolute value of NCPTL_INT_MIN. */
tag = 555666773LL; /* Arbitrary value */
tag = ncptl_func_abs (tag); /* Only nonnegatives values are allowed. */
if (mpi_tag_ub < NCPTL_INT_MAX)
tag %= mpi_tag_ub + 1;
return tag;
}
/* Given an array of task in/out booleans return an MPI
* communicator that represents the "in" tasks. */
static MPI_Comm define_MPI_communicator (char *procflags)
{
MPI_Comm *existing_comm; /* Previously defined MPI communicator */
MPI_Comm new_comm; /* Newly defined MPI communicator */
existing_comm = (MPI_Comm *) ncptl_set_find (communicators, (void *)procflags);
if (existing_comm)
return *existing_comm;
(void) MPI_Comm_split (MPI_COMM_WORLD, (int)procflags[physrank], physrank, &new_comm);
(void) MPI_Errhandler_set (new_comm, mpi_error_handler);
ncptl_set_insert (communicators, (void *)procflags, (void *)&new_comm);
return define_MPI_communicator (procflags);
}
/* Return 1 if a sequence loop will take at least one trip. */
static int conc_seq_nonempty (LOOPBOUNDS *seq)
{
ncptl_int startval; /* Integer version of seq's startval element */
ncptl_int endval; /* Integer version of seq's endval element */
if (seq->integral) {
startval = seq->u.i.startval;
endval = seq->u.i.endval;
}
else {
startval = CONC_DBL2INT (seq->u.d.startval);
endval = CONC_DBL2INT (seq->u.d.endval);
}
switch (seq->comparator) {
case CONC_LEQ:
return startval <= endval;
case CONC_GEQ:
return startval >= endval;
default:
ncptl_fatal ("Internal error -- unknown comparator");
}
return -1; /* Appease idiotic compilers. */
}
/* Initialize a sequence loop. */
static void conc_seq_init (LOOPBOUNDS *seq)
{
if (seq->integral) {
seq->u.i.loopvar = seq->u.i.startval;
seq->u.i.prev_loopvar = seq->u.i.loopvar - 1;
}
else {
seq->u.d.loopvar = seq->u.d.startval;
seq->u.d.prev_loopvar = seq->u.d.loopvar - 1.0;
}
}
/* Return 1 if a sequence loop should continue, 0 when finished. */
static int conc_seq_continue (LOOPBOUNDS *seq)
{
LOOPBOUNDS seq_int; /* Integer equivalent of *seq */
if (seq->integral)
seq_int = *seq;
else {
seq_int.u.i.loopvar = CONC_DBL2INT (seq->u.d.loopvar);
seq_int.u.i.prev_loopvar = CONC_DBL2INT (seq->u.d.prev_loopvar);
seq_int.u.i.endval = CONC_DBL2INT (seq->u.d.endval);
}
if (seq_int.u.i.loopvar == seq_int.u.i.prev_loopvar)
return 0;
switch (seq->comparator) {
case CONC_LEQ:
return seq_int.u.i.loopvar <= seq_int.u.i.endval;
case CONC_GEQ:
return seq_int.u.i.loopvar >= seq_int.u.i.endval;
default:
ncptl_fatal ("Internal error -- unknown comparator");
}
return -1; /* Appease idiotic compilers. */
}
/* Proceed to the next iteration of a sequence loop. */
static void conc_seq_next (LOOPBOUNDS *seq)
{
if (seq->integral) {
seq->u.i.prev_loopvar = seq->u.i.loopvar;
switch (seq->increment) {
case CONC_ADD:
seq->u.i.loopvar += seq->u.i.incval;
break;
case CONC_MULT:
seq->u.i.loopvar *= seq->u.i.incval;
break;
case CONC_DIV:
seq->u.i.loopvar /= seq->u.i.incval;
break;
default:
ncptl_fatal ("Internal error -- unknown incrementer");
}
}
else {
seq->u.d.prev_loopvar = seq->u.d.loopvar;
switch (seq->increment) {
case CONC_ADD:
seq->u.d.loopvar += seq->u.d.incval;
break;
case CONC_MULT:
seq->u.d.loopvar *= seq->u.d.incval;
break;
case CONC_DIV:
seq->u.d.loopvar /= seq->u.d.incval;
break;
default:
ncptl_fatal ("Internal error -- unknown incrementer");
}
}
}
/* Inhibit the compiler from complaining that
* certain variables are defined but not used.
* This function should never be called. */
void conc_mark_variables_used (void)
{
conc_dummy_var.ni = var_bytes_received;
conc_dummy_var.ni = var_msgs_received;
conc_dummy_var.ni = var_bit_errors;
conc_dummy_var.ni = var_total_msgs;
conc_dummy_var.ni = var_msgs_sent;
conc_dummy_var.ni = var_bytes_sent;
conc_dummy_var.ni = var_num_tasks;
conc_dummy_var.ni = var_elapsed_usecs;
conc_dummy_var.ni = var_total_bytes;
conc_dummy_var.ni = pendingrecvbytes;
conc_dummy_var.ni = touch_region_size;
conc_dummy_var.vp = touch_region;
conc_dummy_var.i = within_time_loop;
conc_dummy_var.i = suppress_output;
rank_in_MPI_communicator (MPI_COMM_WORLD, 0);
}
/* Allocate a new event of a given type and return a pointer to it. */
static CONC_EVENT *conc_allocate_event (CONC_EVENT_TYPE type)
{
CONC_EVENT *newevent = (CONC_EVENT *) ncptl_queue_allocate (eventqueue);
newevent->type = type;
return newevent;
}
/* Declare an exit handler that gets called automatically when the
* program terminates, whether successfully or not. */
static void conc_exit_handler (void)
{
if (mpi_is_running)
MPI_Abort (MPI_COMM_WORLD, 1);
}
/* Initialize coNCePTuaL, the messaging layer, and this program itself. */
static void conc_initialize (int argc, char *argv[])
{
/* Variables needed by all C-based backends */
CONC_EVENT * eventlist; /* List of events to execute */
ncptl_int numevents; /* Number of entries in eventlist[] */
int help_only = 0; /* 1=User specified --help; save time by skipping ncptl_init() */
char * argv0 = strrchr(argv[0], '/') ? strrchr(argv[0], '/')+1 : argv[0]; /* Base name of the executable program */
int i; /* Generic loop variable */
/* Declare all of our command-line arguments. */
NCPTL_CMDLINE arguments[] = {
{ NCPTL_TYPE_INT, NULL, "seed", 'S', "Seed for the random-number generator", {0}},
{ NCPTL_TYPE_STRING, NULL, "logfile", 'L', "Log-file template", {0}},
{ NCPTL_TYPE_INT, NULL, "nummsgs", 'n', "Number of messages per trial", {0}},
{ NCPTL_TYPE_INT, NULL, "wups", 'w', "Number of warmup messages", {0}},
{ NCPTL_TYPE_INT, NULL, "bytes", 'b', "Message size in bytes", {0}},
{ NCPTL_TYPE_INT, NULL, "trials", 't', "Number of bisection patterns", {0}},
{ NCPTL_TYPE_INT, NULL, "round", 'r', "Round measurements to the nearest N", {0}}
};
/* Incorporate the complete coNCePTuaL source code as an array
* for use by ncptl_log_write_prologue(). */
char *sourcecode[] = {
"###################################################",
"# Measure random bisection-bandwidth patterns #",
"# By Scott Pakin <pakin@lanl.gov> #",
"# #",
"# Inspired by Hoefler, Schneider, and Lumsdaine's #",
"# \"Multistage Switches are not Crossbars\" paper #",
"###################################################",
"",
"Require language version \"1.5\".",
"",
"nummsgs is \"Number of messages per trial\" and comes from \"--nummsgs\" or \"-n\" with default 100.",
"wups is \"Number of warmup messages\" and comes from \"--wups\" or \"-w\" with default 3.",
"msgsize is \"Message size in bytes\" and comes from \"--bytes\" or \"-b\" with default 1M.",
"numtrials is \"Number of bisection patterns\" and comes from \"--trials\" or \"-t\" with default 5000.",
"rounding is \"Round measurements to the nearest N\" and comes from \"--round\" or \"-r\" with default 5.",
"",
"For each trial in {1, ..., numtrials} {",
" task 0 is assigned to processor 0 then",
" task 0 outputs \"Testing random bisection pattern \" and trial and \"/\" and numtrials then",
" all tasks are assigned to a random processor then",
" tasks src such that src is even send wups msgsize-byte messages to task src+1 then",
" all tasks synchronize then",
" all tasks reset their counters then",
" tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then",
" all tasks synchronize then",
" all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as \"Bandwidth (MiB/s)\"",
"}",
NULL
};
/* Variables specific to the c_mpi backend */
int num_tasks; /* int version of var_num_tasks needed by MPI_Comm_size() */
char * procflags; /* Array of 1s representing an all-task MPI communicator */
MPI_Comm comm_world = MPI_COMM_WORLD; /* Copy of MPI_COMM_WORLD that we can take the address of */
void * attr_val; /* Pointed to the value of MPI_TAG_UB */
int attr_flag = 0; /* true=MPI_TAG_UB was extracted; false=not extracted */
char log_key_str[128]; /* String representing the range of valid MPI tags */
/* As a special case, if the command line contains --help, then skip
* the coNCePTuaL initialization step. */
for (i=1; i<argc; i++)
if (!strcmp(argv[i], "--"))
break;
else
if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-?")) {
argv[1] = "-?"; /* Guaranteed to work, even with getopt() */
help_only = 1;
break;
}
/* Perform various initializations specific to the c_mpi backend. */
/* Initialize MPI. */
(void) MPI_Init(&argc, &argv);
mpi_is_running = 1;
/* Initialize the coNCePTuaL run-time library. */
if (!help_only)
ncptl_init (NCPTL_RUN_TIME_VERSION, argv[0]);
(void) atexit (conc_exit_handler);
/* Initialize the communication routines needed by the c_mpi backend. */
(void) MPI_Errhandler_create ((MPI_Handler_function *)handle_MPI_error, &mpi_error_handler);
(void) MPI_Errhandler_set (MPI_COMM_WORLD, mpi_error_handler);
(void) MPI_Comm_rank(MPI_COMM_WORLD, &physrank);
(void) MPI_Comm_size(MPI_COMM_WORLD, &num_tasks);
var_num_tasks = (ncptl_int) num_tasks;
(void) MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &attr_val, &attr_flag);
mpi_tag_ub = (ncptl_int) (attr_flag ? *(int *)attr_val : 32767);
/* Seed the random-task-number generator. */
random_seed = ncptl_seed_random_task(random_seed, 0);
arguments[0].defaultvalue.intval = random_seed;
/* Generate and broadcast a UUID. */
logfile_uuid = ncptl_log_generate_uuid();
(void) MPI_Bcast ((void *)logfile_uuid, 37, MPI_CHAR, 0, MPI_COMM_WORLD);
/* Plug variables and default values into the NCPTL_CMDLINE structure. */
arguments[0].variable = (CMDLINE_VALUE *) &random_seed;
arguments[0].defaultvalue.intval = random_seed;
arguments[1].variable = (CMDLINE_VALUE *) &logfiletmpl;
arguments[2].variable = (CMDLINE_VALUE *) &var_nummsgs;
arguments[2].defaultvalue.intval = 100LL;
arguments[3].variable = (CMDLINE_VALUE *) &var_wups;
arguments[3].defaultvalue.intval = 3LL;
arguments[4].variable = (CMDLINE_VALUE *) &var_msgsize;
arguments[4].defaultvalue.intval = 1048576LL;
arguments[5].variable = (CMDLINE_VALUE *) &var_numtrials;
arguments[5].defaultvalue.intval = 5000LL;
arguments[6].variable = (CMDLINE_VALUE *) &var_rounding;
arguments[6].defaultvalue.intval = 5LL;
logfiletmpl_default = (char *) ncptl_malloc (strlen(argv0) + 15, 0);
sprintf (logfiletmpl_default, "%s-%%p.log", argv0);
arguments[1].defaultvalue.stringval = logfiletmpl_default;
/* Parse the command line. */
mpi_is_running = 0; /* Don't invoke MPI_Abort() after --help. */
ncptl_parse_command_line (argc, argv, arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE));
if (help_only)
ncptl_fatal ("Internal error in the c_generic backend: failed to exit after giving help");
mpi_is_running = 1;
/* Broadcast the random-task-number generator's seed. */
{
int rndseed_int = (int) random_seed; /* Version of random_seed with int type */
(void) MPI_Bcast ((void *)&rndseed_int, 1, MPI_INT, 0, MPI_COMM_WORLD);
random_seed = (ncptl_int) rndseed_int;
}
(void) ncptl_seed_random_task(random_seed, (ncptl_int)physrank);
/* Establish a mapping from (virtual) task IDs to (physical) ranks. */
procmap = ncptl_allocate_task_map (var_num_tasks);
virtrank = ncptl_physical_to_virtual (procmap, physrank);
/* Perform initializations specific to the c_mpi backend. */
ncptl_log_add_comment ("MPI send routines", "MPI_Send() and MPI_Isend()");
ncptl_log_add_comment ("MPI reduction operation", REDUCE_OPERATION_NAME);
sprintf (log_key_str, "[0, %" NICS "]", mpi_tag_ub);
ncptl_log_add_comment ("MPI tag range", log_key_str);
/* Open the log file and write some standard prologue information to it. */
logstate = ncptl_log_open (logfiletmpl, physrank);
ncptl_log_write_prologue (logstate, argv[0], logfile_uuid, "c_mpi", "C + MPI",
var_num_tasks,
arguments, sizeof(arguments)/sizeof(NCPTL_CMDLINE),
sourcecode);
ncptl_free (logfile_uuid);
/* Allocate a variety of dynamically growing queues. */
eventqueue = ncptl_queue_init (sizeof (CONC_EVENT));
touchedqueue = ncptl_queue_init (sizeof (ncptl_int));
/* Perform initializations specific to the c_mpi backend. */
sendreqQ = ncptl_queue_init (sizeof (MPI_Request));
sendstatQ = ncptl_queue_init (sizeof (MPI_Status));
recvreqQ = ncptl_queue_init (sizeof (MPI_Request));
recvstatQ = ncptl_queue_init (sizeof (MPI_Status));
communicators = ncptl_set_init (ESTIMATED_COMMUNICATORS, var_num_tasks*sizeof(char), sizeof(MPI_Comm));procflags = (char *) ncptl_malloc (var_num_tasks*sizeof(char), 0);
for (i=0; i<var_num_tasks; i++)
procflags[i] = 1;
ncptl_set_insert (communicators, (void *)procflags, (void *)&comm_world);
ncptl_free (procflags);
/****************************************************
* Generated, program-specific initialization code. *
****************************************************/
/* FOR EACH var_trial IN [(['1LL'], 'var_numtrials')]... */
{
LOOPBOUNDS loopbounds[1]; /* List of range descriptions */
ncptl_int rangenum; /* Current offset into loopbounds[] */
ncptl_int initial_vals[1]; /* Cache of the initial, enumerated values */
ncptl_int final_val; /* Cache of the final value */
/* Write range 0's loop bounds, "next" function, and termination function to loopbounds[0]. */
loopbounds[0].list_comp = NULL;
initial_vals[0] = 1LL;
final_val = var_numtrials;
loopbounds[0].integral = 1;
loopbounds[0].u.i.startval = initial_vals[0];
loopbounds[0].u.i.endval = final_val;
loopbounds[0].comparator = initial_vals[0]<=final_val ? CONC_LEQ : CONC_GEQ;
loopbounds[0].increment = CONC_ADD;
loopbounds[0].u.i.incval = initial_vals[0]<=final_val ? 1LL : -1LL;
/* Now that we've defined all of our ranges we iterate over each range
* and each element within each range. */
for (rangenum=0LL; rangenum<1; rangenum++) {
LOOPBOUNDS * thisrange = &loopbounds[rangenum]; /* Current range */
if (conc_seq_nonempty (thisrange))
for (conc_seq_init (thisrange);
conc_seq_continue (thisrange);
conc_seq_next (thisrange)) {
ncptl_int var_trial;
if (thisrange->list_comp == NULL)
/* The loopvar field contains the value for var_trial. */
var_trial = thisrange->integral ? thisrange->u.i.loopvar : CONC_DBL2INT(thisrange->u.d.loopvar);
else
/* The queue of list-comprehension values provides the value for var_trial. */
var_trial = *(ncptl_int *)ncptl_queue_pop(thisrange->list_comp);
{
/* TASK 0LL ARE ASSIGNED PROCESSOR 0LL */
{
/* In the following, "task ID" implies a virtual rank in the
* computation while "processor" implies a physical rank. */
ncptl_int virtID = 0LL; /* Task ID */
procmap = ncptl_conditionally_copy_task_map (procmap);
virtrank = ncptl_assign_processor (virtID, 0LL, procmap, physrank);
}
/* THEN... */
/* TASK 0LL OUTPUTS ['"Testing random bisection pattern "', 'var_trial', '"/"', 'var_numtrials'] */
if ((0LL) == virtrank) { /* TASK 0LL */
/* The current coNCePTuaL statement applies to our task. */
CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
thisev->s.code.number = 0;
thisev->s.code.procmap = NULL;
thisev->s.code.var_numtrials = var_numtrials;
thisev->s.code.var_trial = var_trial;
}
/* THEN... */
/* ALL TASKS ARE ASSIGNED PROCESSOR [random] */
{
/* In the following, "task ID" implies a virtual rank in the
* computation while "processor" implies a physical rank. */
ncptl_int virtID; /* Task ID */
procmap = ncptl_conditionally_copy_task_map (procmap);
for (virtID=0; virtID<var_num_tasks; virtID++) {
virtrank = ncptl_assign_processor (virtID, ncptl_random_task(0, var_num_tasks-1, -1), procmap, physrank);
}
}
/* THEN... */
/* TASKS var_src SUCH THAT ((var_src)&1)==0 RECEIVES FROM TASK (var_src)+(1LL) */
{
ncptl_int var_src;
/* Loop over all tasks to see which will send to us. */
for (var_src=0; var_src<var_num_tasks; var_src++)
if (((var_src)&1)==0) {
/* var_src now represents one of the tasks that will send to us. */
ncptl_int ivar_a_task = var_src;
if (virtrank == ((var_src)+(1LL))) {
/* In this scope, we must be a message recipient. */
/* Prepare to receive var_wups messages. */
ncptl_int numreps = var_wups; /* Number of messages */
if (numreps > 1LL) {
CONC_EVENT * repeatev; /* Event specifying the number of repetitions to perform */
repeatev = conc_allocate_event (EV_REPEAT);
repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
repeatev->s.rep.numreps = numreps;
}
/* Ensure we have at least one message to receive. */
if (numreps > 0LL) {
CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
/* Fill in all of the fields of a receive-event structure. */
thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_a_task);
thisev->s.recv.size = var_msgsize;
thisev->s.recv.alignment = 0LL;
thisev->s.recv.misaligned = 0;
thisev->s.recv.touching = 0;
thisev->s.recv.verification = 0;
thisev->s.recv.tag = 0LL;
thisev->s.recv.pendingsends = pendingsends;
thisev->s.recv.pendingrecvs = pendingrecvs;
thisev->s.recv.bufferofs = 0LL;
thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
thisev->s.recv.buffer = NULL;
thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
}
}
}
}
/* TASKS var_src SUCH THAT ((var_src)&1)==0 SENDS TO TASK (var_src)+(1LL) */
{
ncptl_int var_src = virtrank;
if (((var_src)&1)==0) { /* TASKS var_src SUCH THAT ((var_src)&1)==0 */
/* The current coNCePTuaL statement applies to our task. */
ncptl_int virtdest = (var_src)+(1LL);
if (virtdest>=0 && virtdest<var_num_tasks) {
/* In this scope, (var_src)+(1LL) represents a single receiver. */
/* Prepare to send var_wups messages. */
ncptl_int numreps = var_wups; /* Number of messages */
if (numreps > 1LL) {
CONC_EVENT * repeatev; /* Event specifying the number of repetitions to perform */
repeatev = conc_allocate_event (EV_REPEAT);
repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
repeatev->s.rep.numreps = numreps;
}
/* Ensure we have at least one message to send. */
if (numreps > 0LL) {
CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
if (virtrank == ((var_src)+(1LL)))
ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 21 of the source code", virtrank);
/* Fill in all of the fields of a send-event structure. */
thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_src)+(1LL));
thisev->s.send.size = var_msgsize;
thisev->s.send.alignment = 0LL;
thisev->s.send.misaligned = 0;
thisev->s.send.touching = 0;
thisev->s.send.verification = 0;
thisev->s.send.tag = 0LL;
thisev->s.send.pendingsends = pendingsends;
thisev->s.send.pendingrecvs = pendingrecvs;
thisev->s.send.bufferofs = 0LL;
thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
thisev->s.send.buffer = NULL;
thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
}
}
}
}
/* THEN... */
/* ALL TASKS SYNCHRONIZE */
{ /* ALL TASKS */
/* The current coNCePTuaL statement applies to our task. */
CONC_EVENT *thisev = conc_allocate_event (EV_SYNC);
thisev->s.sync.communicator = MPI_COMM_WORLD;
}
/* THEN... */
/* ALL TASKS RESET THEIR COUNTERS */
{ /* ALL TASKS */
/* The current coNCePTuaL statement applies to our task. */
(void) conc_allocate_event (EV_RESET);
}
/* THEN... */
/* TASKS var_src SUCH THAT ((var_src)&1)==0 RECEIVES FROM TASK (var_src)+(1LL) */
{
ncptl_int var_src;
/* Loop over all tasks to see which will send to us. */
for (var_src=0; var_src<var_num_tasks; var_src++)
if (((var_src)&1)==0) {
/* var_src now represents one of the tasks that will send to us. */
ncptl_int ivar_b_task = var_src;
if (virtrank == ((var_src)+(1LL))) {
/* In this scope, we must be a message recipient. */
/* Prepare to receive var_nummsgs messages. */
ncptl_int numreps = var_nummsgs; /* Number of messages */
if (numreps > 1LL) {
CONC_EVENT * repeatev; /* Event specifying the number of repetitions to perform */
repeatev = conc_allocate_event (EV_REPEAT);
repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
repeatev->s.rep.numreps = numreps;
}
/* Ensure we have at least one message to receive. */
if (numreps > 0LL) {
CONC_EVENT *thisev = conc_allocate_event (EV_RECV);
/* Fill in all of the fields of a receive-event structure. */
thisev->s.recv.source = ncptl_virtual_to_physical (procmap, ivar_b_task);
thisev->s.recv.size = var_msgsize;
thisev->s.recv.alignment = 0LL;
thisev->s.recv.misaligned = 0;
thisev->s.recv.touching = 0;
thisev->s.recv.verification = 0;
thisev->s.recv.tag = 0LL;
thisev->s.recv.pendingsends = pendingsends;
thisev->s.recv.pendingrecvs = pendingrecvs;
thisev->s.recv.bufferofs = 0LL;
thisev->s.recv.buffernum = thisev->s.recv.verification ? pendingsends+pendingrecvs : 0;
(void) ncptl_malloc_message (thisev->s.recv.size+thisev->s.recv.bufferofs, thisev->s.recv.alignment, thisev->s.recv.buffernum, thisev->s.recv.misaligned);
thisev->s.recv.buffer = NULL;
thisev->s.recv.tag = map_tag_into_MPI_range (thisev->s.recv.tag);
}
}
}
}
/* TASKS var_src SUCH THAT ((var_src)&1)==0 SENDS TO TASK (var_src)+(1LL) */
{
ncptl_int var_src = virtrank;
if (((var_src)&1)==0) { /* TASKS var_src SUCH THAT ((var_src)&1)==0 */
/* The current coNCePTuaL statement applies to our task. */
ncptl_int virtdest = (var_src)+(1LL);
if (virtdest>=0 && virtdest<var_num_tasks) {
/* In this scope, (var_src)+(1LL) represents a single receiver. */
/* Prepare to send var_nummsgs messages. */
ncptl_int numreps = var_nummsgs; /* Number of messages */
if (numreps > 1LL) {
CONC_EVENT * repeatev; /* Event specifying the number of repetitions to perform */
repeatev = conc_allocate_event (EV_REPEAT);
repeatev->s.rep.end_event = ncptl_queue_length(eventqueue);
repeatev->s.rep.numreps = numreps;
}
/* Ensure we have at least one message to send. */
if (numreps > 0LL) {
CONC_EVENT *thisev = conc_allocate_event (EV_SEND);
if (virtrank == ((var_src)+(1LL)))
ncptl_fatal ("Send-to-self deadlock encountered on task %d in line 24 of the source code", virtrank);
/* Fill in all of the fields of a send-event structure. */
thisev->s.send.dest = ncptl_virtual_to_physical (procmap, (var_src)+(1LL));
thisev->s.send.size = var_msgsize;
thisev->s.send.alignment = 0LL;
thisev->s.send.misaligned = 0;
thisev->s.send.touching = 0;
thisev->s.send.verification = 0;
thisev->s.send.tag = 0LL;
thisev->s.send.pendingsends = pendingsends;
thisev->s.send.pendingrecvs = pendingrecvs;
thisev->s.send.bufferofs = 0LL;
thisev->s.send.buffernum = thisev->s.send.verification ? pendingsends+pendingrecvs : 0;
(void) ncptl_malloc_message (thisev->s.send.size+thisev->s.send.bufferofs, thisev->s.send.alignment, thisev->s.send.buffernum, thisev->s.send.misaligned);
thisev->s.send.buffer = NULL;
thisev->s.send.tag = map_tag_into_MPI_range (thisev->s.send.tag);
}
}
}
}
/* THEN... */
/* ALL TASKS SYNCHRONIZE */
{ /* ALL TASKS */
/* The current coNCePTuaL statement applies to our task. */
CONC_EVENT *thisev = conc_allocate_event (EV_SYNC);
thisev->s.sync.communicator = MPI_COMM_WORLD;
}
/* THEN... */
/* ALL TASKS LOGS "Bandwidth (MiB/s)" */
{ /* ALL TASKS */
/* The current coNCePTuaL statement applies to our task. */
CONC_EVENT *thisev = conc_allocate_event (EV_CODE);
thisev->s.code.number = 1;
thisev->s.code.procmap = NULL;
thisev->s.code.var_rounding = var_rounding;
}
}
}
}
}
/* Begin a new top-level statement. */
(void) conc_allocate_event (EV_NEWSTMT);
/*************************
* More boilerplate code *
*************************/
/* Abort if the program will terminate with pending messages. */
if (pendingsends && pendingrecvs)
ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s and %" NICS " asynchronous %s",
pendingsends, pendingsends==1LL ? "send" : "sends",
pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
else
if (pendingsends)
ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
pendingsends, pendingsends==1LL ? "send" : "sends");
else
if (pendingrecvs)
ncptl_fatal("Neglected to await the completion of %" NICS " asynchronous %s",
pendingrecvs, pendingrecvs==1LL ? "receive" : "receives");
/* Allocate memory for non-unique messages and asynchronous
* message handles now that we know how much memory we need
* to allocate. */
eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
numevents = ncptl_queue_length (eventqueue);
sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
for (i=0; i<numevents; i++) {
CONC_EVENT *thisev = &eventlist[i]; /* Cache of the current event */
switch (thisev->type) {
case EV_SEND:
if (!thisev->s.send.buffer)
thisev->s.send.buffer = ncptl_malloc_message (thisev->s.send.bufferofs + thisev->s.send.size,
thisev->s.send.alignment,
thisev->s.send.buffernum,
thisev->s.send.misaligned);
if (thisev->s.send.verification)
ncptl_fill_buffer (CONC_GETBUFPTR(send), thisev->s.send.bufferofs + thisev->s.send.size, -1);
break;
case EV_RECV:
if (!thisev->s.recv.buffer)
thisev->s.recv.buffer = ncptl_malloc_message (thisev->s.recv.bufferofs + thisev->s.recv.size,
thisev->s.recv.alignment,
thisev->s.recv.buffernum,
thisev->s.recv.misaligned);
if (thisev->s.recv.verification)
ncptl_fill_buffer (CONC_GETBUFPTR(recv), thisev->s.recv.bufferofs + thisev->s.recv.size, -1);
break;
default:
break;
}
}
}
/* Process a subset of the events in a given event list. */
static void conc_process_events (CONC_EVENT *eventlist,
ncptl_int firstev, ncptl_int lastev, ncptl_int numreps)
{
CONC_EVENT * thisev; /* Cache of the current event */
CONC_EVENT * thisev_first = &eventlist[firstev]; /* Cache of the first event */
ncptl_int i; /* Iterate over events. */
ncptl_int j; /* Iterate over repetitions. */
/* Declarations specific to the c_mpi backend */
MPI_Status status; /* Not needed but required by MPI_Recv() */
/* Process from event firstev to event lastev (both inclusive). */
for (j=numreps; j>0; j--)
for (i=firstev, thisev=thisev_first; i<=lastev; i++, thisev++) {
/* Declare variables needed by all C-based backends. */
/* Process a single event. */
switch (thisev->type) {
case EV_SEND:
/* Synchronous send */
(void) MPI_Send (CONC_GETBUFPTR(send),
(int)thisev->s.send.size, MPI_BYTE,
(int)thisev->s.send.dest, (int)thisev->s.send.tag, MPI_COMM_WORLD);
var_total_bytes += thisev->s.send.size;
break;
case EV_RECV:
/* Synchronous receive */
(void) MPI_Recv (CONC_GETBUFPTR(recv),
(int)thisev->s.recv.size, MPI_BYTE,
(int)thisev->s.recv.source, (int)thisev->s.recv.tag,
MPI_COMM_WORLD, &status);
var_total_bytes += thisev->s.recv.size;
break;
case EV_SYNC:
/* Synchronize a subset of the tasks. */
(void) MPI_Barrier (thisev->s.sync.communicator);
break;
case EV_RESET:
/* Reset all of the counters exported to coNCePTuaL programs. */
var_total_bytes = 0;
starttime = ncptl_time();
break;
case EV_REPEAT:
/* Repeatedly perform the next batch of events. */
conc_process_events (eventlist, i+1, thisev->s.rep.end_event, thisev->s.rep.numreps);
i = thisev->s.rep.end_event;
thisev = &eventlist[i];
break;
case EV_NEWSTMT:
/* Begin a new table in the log file. */
if (!suppress_output) {
uint64_t stop_elapsed_usecs = ncptl_time();
var_elapsed_usecs = stop_elapsed_usecs - starttime;
ncptl_log_commit_data (logstate);
starttime += ncptl_time() - stop_elapsed_usecs;
}
break;
case EV_CODE:
/* Execute an arbitrary piece of code. */
switch (thisev->s.code.number) {
case 0:
/* TASK 0LL OUTPUTS ['"Testing random bisection pattern "', 'var_trial', '"/"', 'var_numtrials'] */
if (!suppress_output) {
uint64_t stop_elapsed_usecs = ncptl_time();
var_elapsed_usecs = stop_elapsed_usecs - starttime;
printf ("%s%.10lg%s%.10lg\n", "Testing random bisection pattern ", (double)thisev->s.code.var_trial, "/", (double)thisev->s.code.var_numtrials);
fflush (stdout);
starttime += ncptl_time() - stop_elapsed_usecs;
}
break;
case 1:
/* ALL TASKS LOGS "Bandwidth (MiB/s)" */
if (!suppress_output) {
uint64_t stop_elapsed_usecs = ncptl_time();
var_elapsed_usecs = stop_elapsed_usecs - starttime;
ncptl_log_write (logstate, 0, "Bandwidth (MiB/s)", NCPTL_FUNC_HISTOGRAM, 0.0, ((double)thisev->s.code.var_rounding)*(ncptl_dfunc_round(((((double)var_total_bytes)*(1000000.0))/(((double)var_elapsed_usecs)*(1048576.0)))/((double)thisev->s.code.var_rounding))));
starttime += ncptl_time() - stop_elapsed_usecs;
}
break;
default:
/* The C code generation module must be broken. */
ncptl_fatal ("Internal error: unknown EV_CODE block %" NICS, thisev->s.code.number);
break;
}
break;
default:
/* The c_generic backend or the c_mpi backend must be broken. */
ncptl_fatal ("Internal error: unknown event type %d", thisev->type);
break;
}
}
}
/* Finish up cleanly and return a status code. */
static int conc_finalize (void)
{
int exitcode = 0; /* Program exit code (to pass to exit()) */
/* Declarations specific to the c_mpi backend */
int mpiresult; /* Return code from MPI_Finalize() */
char log_key_str[128]; /* String representing the range of valid MPI tags */
/* Finalization code specific to the c_mpi backend */
log_key_str[0] = '\0';
if (conc_mcast_tallies[CONC_MCAST_MPI_BCAST] > 0) {
char onefuncstr[50];
sprintf (onefuncstr, "%sMPI_Bcast()*%" NICS,
log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_BCAST]);
strcat (log_key_str, onefuncstr);
}
if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL] > 0) {
char onefuncstr[50];
sprintf (onefuncstr, "%sMPI_Alltoall()*%" NICS,
log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALL]);
strcat (log_key_str, onefuncstr);
}
if (conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV] > 0) {
char onefuncstr[50];
sprintf (onefuncstr, "%sMPI_Alltoallv()*%" NICS,
log_key_str[0] == '\0' ? "" : " ", conc_mcast_tallies[CONC_MCAST_MPI_ALLTOALLV]);
strcat (log_key_str, onefuncstr);
}
if (log_key_str[0] != '\0')
ncptl_log_add_comment ("Multicast functions used (statically)", log_key_str);
/* Write a standard epilogue to the log file. */
ncptl_log_commit_data (logstate);
ncptl_log_write_epilogue (logstate);
ncptl_log_close (logstate);
/* Inform the run-time library that it's no longer needed. */
ncptl_queue_empty (eventqueue);
ncptl_free (eventqueue);
ncptl_finalize();
/* Finalization code specific to the c_mpi backend */
mpiresult = MPI_Finalize();
mpi_is_running = 0;
exitcode = mpiresult!=MPI_SUCCESS;
/* Return an exit status code. */
return exitcode;
}
/*************************************************************************/
/***************************** MAIN ROUTINE ******************************/
/*************************************************************************/
/* Program execution starts here. */
int main (int argc, char *argv[])
{
/* Declare variables needed by all C-based backends. */
CONC_EVENT * eventlist; /* List of events to execute */
ncptl_int numevents; /* Number of entries in eventlist[] */
/* ----- Initialization ----- */
conc_initialize (argc, argv);
eventlist = (CONC_EVENT *) ncptl_queue_contents (eventqueue, 0);
numevents = ncptl_queue_length (eventqueue);
sendrequests = (MPI_Request *) ncptl_queue_contents (sendreqQ, 0);
sendstatuses = (MPI_Status *) ncptl_queue_contents (sendstatQ, 0);
recvrequests = (MPI_Request *) ncptl_queue_contents (recvreqQ, 0);
recvstatuses = (MPI_Status *) ncptl_queue_contents (recvstatQ, 0);
starttime = ncptl_time();
/* ----- Event-list processing ----- */
conc_process_events (eventlist, 0, numevents-1, 1);
/* ----- Finalization ----- */
return conc_finalize();
}
###################################################
# Measure random bisection-bandwidth patterns #
# By Scott Pakin <pakin@lanl.gov> #
# #
# Inspired by Hoefler, Schneider, and Lumsdaine's #
# "Multistage Switches are not Crossbars" paper #
###################################################
Require language version "1.5".
nummsgs is "Number of messages per trial" and comes from "--nummsgs" or "-n" with default 100.
wups is "Number of warmup messages" and comes from "--wups" or "-w" with default 3.
msgsize is "Message size in bytes" and comes from "--bytes" or "-b" with default 1M.
numtrials is "Number of bisection patterns" and comes from "--trials" or "-t" with default 5000.
rounding is "Round measurements to the nearest N" and comes from "--round" or "-r" with default 5.
For each trial in {1, ..., numtrials} {
task 0 is assigned to processor 0 then
task 0 outputs "Testing random bisection pattern " and trial and "/" and numtrials then
all tasks are assigned to a random processor then
tasks src such that src is even send wups msgsize-byte messages to task src+1 then
all tasks synchronize then
all tasks reset their counters then
tasks src such that src is even send nummsgs msgsize-byte messages to task src+1 then
all tasks synchronize then
all tasks log a histogram of rounding*round(total_bytes*1E6/(elapsed_usecs*1M)/rounding) as "Bandwidth (MiB/s)"