model-net.h 15.7 KB
Newer Older
1
/*
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
4
 *
5 6 7 8 9
 */

#ifndef MODELNET_H
#define MODELNET_H

Jonathan Jenkins's avatar
Jonathan Jenkins committed
10 11 12 13
#ifdef __cplusplus
extern "C" {
#endif

14 15 16 17 18
#include <ross.h>
#include <codes/lp-type-lookup.h>
#include <codes/configuration.h>
#include <codes/lp-io.h>
#include <codes/codes-mapping-context.h>
19
#include <stdint.h>
20

21 22 23 24 25
#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif

26 27
#define PULL_MSG_SIZE 128

28
#define MAX_NAME_LENGTH 256
29 30
#define CATEGORY_NAME_MAX 16
#define CATEGORY_MAX 12
31

32
// simple deprecation attribute hacking
33 34 35 36 37 38
#if !defined(DEPRECATED)
#  if defined(__GNUC__) || defined(__GNUG__) || defined(__clang__)
#    define DEPRECATED __attribute__((deprecated))
#  else
#    define DEPRECATED
#  endif
39 40
#endif

41 42 43
/* HACK: there is currently no scheduling fidelity across multiple
 * model_net_event calls. Hence, problems arise when some LP sends multiple
 * messages as part of an event and expects FCFS ordering. A proper fix which
44
 * involves model-net LP-level scheduling of requests is ideal, but not
45
 * feasible for now (would basically have to redesign model-net), so expose
46 47
 * explicit start-sequence and stop-sequence markers as a workaround
 */
48
extern int mn_in_sequence;
49 50
extern tw_stime mn_msg_offset;
#define MN_START_SEQ() do {\
51
    mn_in_sequence = 1; \
52 53 54
    mn_msg_offset = 0.0; \
} while (0)
#define MN_END_SEQ() do {\
55
    mn_in_sequence = 0;\
56 57 58
} while (0)


59
typedef struct mn_stats mn_stats;
60

61 62 63 64 65
// use the X-macro to get types and names rolled up into one structure
// format: { enum vals, config name, internal lp name, lp method struct}
// last value is sentinel
#define NETWORK_DEF \
    X(SIMPLENET, "modelnet_simplenet", "simplenet", &simplenet_method)\
66
    X(SIMPLEP2P, "modelnet_simplep2p", "simplep2p", &simplep2p_method)\
67
    X(TORUS,     "modelnet_torus",     "torus",     &torus_method)\
68
    X(SLIMFLY,   "modelnet_slimfly",   "slimfly",   &slimfly_method)\
69
    X(SLIMFLY_ROUTER, "modelnet_slimfly_router", "slimfly_router", &slimfly_router_method)\
70
    X(FATTREE,   "modelnet_fattree",   "fattree",   &fattree_method)\
71
    X(DRAGONFLY, "modelnet_dragonfly", "dragonfly", &dragonfly_method)\
72
    X(DRAGONFLY_ROUTER, "modelnet_dragonfly_router", "dragonfly_router", &dragonfly_router_method)\
73 74
    X(DRAGONFLY_CUSTOM, "modelnet_dragonfly_custom", "dragonfly_custom", &dragonfly_custom_method)\
    X(DRAGONFLY_CUSTOM_ROUTER, "modelnet_dragonfly_custom_router", "dragonfly_custom_router", &dragonfly_custom_router_method)\
75
    X(LOGGP,     "modelnet_loggp",     "loggp",     &loggp_method)\
76 77
    X(EXPRESS_MESH, "modelnet_express_mesh", "express_mesh", &express_mesh_method)\
    X(EXPRESS_MESH_ROUTER, "modelnet_express_mesh_router", "express_mesh_router", &express_mesh_router_method)\
78 79
    X(DRAGONFLY_PLUS, "modelnet_dragonfly_plus", "dragonfly_plus", &dragonfly_plus_method)\
    X(DRAGONFLY_PLUS_ROUTER, "modelnet_dragonfly_plus_router", "dragonfly_plus_router", &dragonfly_plus_router_method)\
80 81
    X(DRAGONFLY_DALLY, "modelnet_dragonfly_dally", "dragonfly_dally", &dragonfly_dally_method)\
    X(DRAGONFLY_DALLY_ROUTER, "modelnet_dragonfly_dally_router", "dragonfly_dally_router", &dragonfly_dally_router_method)\
82 83 84
    X(MAX_NETS,  NULL,                 NULL,        NULL)

#define X(a,b,c,d) a,
85 86
enum NETWORKS
{
87
    NETWORK_DEF
88
};
89 90
#undef X

91 92
// message parameter types
enum msg_param_type {
93
    // parameters for modelnet-scheduler (priorities and such)
94
    MN_MSG_PARAM_SCHED,
95 96 97
    // parameter allowing the explicit setting of a messages start time (useful
    // for routing scenarios in which a single message is implemented in terms
    // of multiple model-net events, such as for model_net_pull_*event's)
98
    MN_MSG_PARAM_START_TIME,
99 100 101
    MAX_MN_MSG_PARAM_TYPES
};

102 103 104 105 106
// MN_MSG_PARAM_MSG_START_TIME parameter types (only one for the time being)
enum msg_param_start_time {
    MN_MSG_PARAM_START_TIME_VAL
};

107 108 109 110 111
// return type for model_net_*event calls, to be passed into RC
// currently is just an int, but in the future may indicate whether lp-io was
// called etc.
typedef int model_net_event_return;

112 113 114 115
// network identifiers (both the config lp names and the model-net internal
// names)
extern char * model_net_lp_config_names[];
extern char * model_net_method_names[];
116

117 118 119 120
// request structure that gets passed around (by the model-net implementation,
// not the user)
typedef struct model_net_request {
    tw_lpid  final_dest_lp;
121
    tw_lpid  dest_mn_lp; // destination modelnet lp
122
    tw_lpid  src_lp;
123 124
    // time the source event was called
    tw_stime msg_start_time;
125
    uint64_t msg_size;
126
    uint64_t pull_size;
127
    uint64_t packet_size;
128 129 130
    // unique message id set, *at the modelnet LP*, for each message the
    // modelnet LP processes
    uint64_t msg_id;
131 132
    int      net_id;
    int      is_pull;
Nikhil's avatar
Nikhil committed
133
    int      queue_offset;
134 135 136 137 138
    int      remote_event_size;
    int      self_event_size;
    char     category[CATEGORY_NAME_MAX];
} model_net_request;

139 140 141 142 143 144 145 146 147 148 149 150 151
/* data structure for tracking network statistics */
struct mn_stats
{
    char category[CATEGORY_NAME_MAX];
    long send_count;
    long send_bytes;
    tw_stime send_time;
    long recv_count;
    long recv_bytes;
    tw_stime recv_time;
    long max_event_size;
};

152
/* Registers all model-net LPs in ROSS. Should be called after
153 154
 * configuration_load, but before codes_mapping_setup */
void model_net_register();
155

156 157 158 159 160 161 162 163
/* Configures all model-net LPs based on the CODES configuration, and returns
 * ids to address the different types by.
 *
 * id_count - the output number of networks
 *
 * return - the set of network IDs, indexed in the order given by the
 * modelnet_order configuration parameter */
int* model_net_configure(int *id_count);
164

165 166 167
/* Sets up a sampling loop for model net events. The sampling data provided by
 * each modelnet lp is model-defined. This is a PE-wide setting. Data is sent
 * to LP-IO with the category modelnet-samples */
168
void model_net_enable_sampling(tw_stime interval, tw_stime end);
169 170 171 172

/* Returns 1 if modelnet is performing sampling, 0 otherwise */
int model_net_sampling_enabled(void);

173
/* Initialize/configure the network(s) based on the CODES configuration.
174 175
 * returns an array of the network ids, indexed in the order given by the
 * modelnet_order configuration parameter
176 177
 * OUTPUT id_count - the output number of networks */
int* model_net_set_params(int *id_count);
178

179 180
/* utility function to get the modelnet ID post-setup */
int model_net_get_id(char *net_name);
181 182 183 184

/* This event does a collective operation call for model-net */
void model_net_event_collective(
    int net_id,
185
    char const * category,
186 187
    int message_size,
    int remote_event_size,
188
    void const * remote_event,
189 190 191 192 193 194 195 196
    tw_lp *sender);

/* reverse event of the collective operation call */
void model_net_event_collective_rc(
        int net_id,
        int message_size,
        tw_lp *sender);

197
/* allocate and transmit a new event that will pass through model_net to
198 199
 * arrive at its destination:
 *
200 201
 * - net_id: the type of network to send this message through. The set of
 *   net_id's is given by model_net_configure.
202 203 204 205 206 207 208 209 210 211 212 213
 * - category: category name to associate with this communication
 *   - OPTIONAL: callers can set this to NULL if they don't want to use it,
 *     and model_net methods can ignore it if they don't support it
 * - final_dest_lp: the LP that the message should be delivered to.
 *   - NOTE: this is _not_ the LP of an underlying network method (for
 *     example, it is not a torus or dragonfly LP), but rather the LP of an
 *     MPI process or storage server that you are transmitting to.
 * - message_size: this is the size of the message (in bytes) that modelnet
 *     will simulate transmitting to the final_dest_lp.  It can be any size
 *     (i.e. it is not constrained by transport packet size).
 * - remote_event_size: this is the size of the ROSS event structure that
 *     will be delivered to the final_dest_lp.
214 215 216
 * - remote_event: pointer to data to be used as the remote event message. When
 *   the message payload (of size message_size) has been fully delivered to the
 *   destination (given by final_dest_lp), this event will be issued.
217 218 219 220 221 222 223
 * - self_event_size: this is the size of the ROSS event structure that will
 *     be delivered to the calling LP once local completion has occurred for
 *     the network transmission.
 *     - NOTE: "local completion" in this sense means that model_net has
 *       transmitted the data off of the local node, but it does not mean that
 *       the data has been (or even will be) delivered.  Once this event is
 *       delivered the caller is free to re-use its buffer.
224 225 226 227
 * - self_event: pointer to data to be used as the self event message. When the
 *   message payload (of size message_size) has been fully sent from the
 *   sender's point of view (e.g. the corresponding NIC has sent out all
 *   packets for this message), the event will be issued to the sender.
228 229
 * - sender: pointer to the tw_lp structure of the API caller.  This is
 *     identical to the sender argument to tw_event_new().
230
 *
231 232 233 234
 * The modelnet LP used for communication is determined by the default CODES
 * map context (see codes-base, codes/codes-mapping-context.h), using net_id
 * to differentiate different model types. Note that the map context is used
 * when calculating *both* sender and receiver modelnet LPs
235 236
 */
// first argument becomes the network ID
237
model_net_event_return model_net_event(
238
    int net_id,
239 240 241
    char const * category,
    tw_lpid final_dest_lp,
    uint64_t message_size,
242
    tw_stime offset,
243
    int remote_event_size,
244
    void const * remote_event,
245
    int self_event_size,
246
    void const * self_event,
247
    tw_lp *sender);
248 249 250 251 252
/*
 * See model_net_event for a general description.
 *
 * Unlike model_net_event, this function uses the annotation to differentiate
 * multiple modelnet LPs with the same type but different annotation. The caller
253 254
 * annotation is not consulted here. The corresponding CODES map context is
 * CODES_MCTX_GROUP_MODULO with the supplied annotation arguments.
255
 */
256
model_net_event_return model_net_event_annotated(
257
        int net_id,
258
        char const * annotation,
259 260 261
        char const * category,
        tw_lpid final_dest_lp,
        uint64_t message_size,
262 263
        tw_stime offset,
        int remote_event_size,
264
        void const * remote_event,
265
        int self_event_size,
266
        void const * self_event,
267
        tw_lp *sender);
268 269 270 271 272 273
/*
 * See model_net_event for a general description.
 *
 * This variant uses CODES map contexts to calculate the sender and receiver
 * modelnet LPs
 */
274
model_net_event_return model_net_event_mctx(
275 276 277
        int net_id,
        struct codes_mctx const * send_map_ctx,
        struct codes_mctx const * recv_map_ctx,
278 279 280
        char const * category,
        tw_lpid final_dest_lp,
        uint64_t message_size,
281 282 283 284 285 286 287
        tw_stime offset,
        int remote_event_size,
        void const * remote_event,
        int self_event_size,
        void const * self_event,
        tw_lp *sender);

288

289 290
/* model_net_find_local_device()
 *
291 292 293
 * returns the LP id of the network card attached to the calling LP using the
 * default CODES mapping context if ignore_annotations is true, and
 * CODES_MCTX_GROUP_MODULO with the supplied annotation parameters otherwise
294
 */
295 296 297 298
tw_lpid model_net_find_local_device(
        int          net_id,
        const char * annotation,
        int          ignore_annotations,
299
        tw_lpid      sender_gid);
300

301 302 303 304 305 306
/* same as ^, except use the supplied mapping context */
tw_lpid model_net_find_local_device_mctx(
        int net_id,
        struct codes_mctx const * map_ctx,
        tw_lpid sender_gid);

307 308
int model_net_get_msg_sz(int net_id);

309 310 311 312 313 314 315 316
/* model_net_event_rc()
 *
 * This function does reverse computation for the model_net_event_new()
 * function.
 * - sender: pointer to the tw_lp structure of the API caller.  This is
 *   identical to the sender argument to tw_event_new().
 */
/* NOTE: we may end up needing additoinal arguments here to track state for
317
 * reverse computation; add as needed
318
 */
319
DEPRECATED
320 321 322
void model_net_event_rc(
    int net_id,
    tw_lp *sender,
323
    uint64_t message_size);
324

325 326 327 328 329 330
/* This function replaces model_net_event_rc, and will replace the name later
 * on. The num_rng_calls argument is the return value of model_net_*event*
 * calls */
void model_net_event_rc2(
        tw_lp *sender,
        model_net_event_return const * ret);
331 332 333 334 335 336 337 338 339 340

/* Issue a 'pull' from the memory of the destination LP, without
 * requiring the destination LP to do event processing. This is meant as a
 * simulation-based abstraction of RDMA. A control packet will be sent to the
 * destination LP, the payload will be sent back to the requesting LP, and the
 * requesting LP will be issued it's given completion event.
 *
 * Parameters are largely the same as model_net_event, with the following
 * exceptions:
 * - final_dest_lp is the lp to pull data from
341
 * - self_event_size, self_event are applied at the requester upon receipt of
342 343
 *   the payload from the dest
 */
344
model_net_event_return model_net_pull_event(
345
        int net_id,
346
        char const *category,
347 348 349 350
        tw_lpid final_dest_lp,
        uint64_t message_size,
        tw_stime offset,
        int self_event_size,
351
        void const *self_event,
352
        tw_lp *sender);
353
model_net_event_return model_net_pull_event_annotated(
354
        int net_id,
355 356
        char const * annotation,
        char const *category,
357 358 359 360
        tw_lpid final_dest_lp,
        uint64_t message_size,
        tw_stime offset,
        int self_event_size,
361
        void const *self_event,
362
        tw_lp *sender);
363
model_net_event_return model_net_pull_event_mctx(
364
        int net_id,
365 366
        struct codes_mctx const * send_map_ctx,
        struct codes_mctx const * recv_map_ctx,
367 368 369 370 371 372 373
        char const *category,
        tw_lpid final_dest_lp,
        uint64_t message_size,
        tw_stime offset,
        int self_event_size,
        void const *self_event,
        tw_lp *sender);
374

375
DEPRECATED
376 377 378 379
void model_net_pull_event_rc(
        int net_id,
        tw_lp *sender);

380 381 382 383 384 385 386 387 388 389 390
/*
 * Set message-specific parameters
 * type     - overall type (see msg_param_type)
 * sub_type - type of parameter specific to type. This is intended to be
 *            an enum for each of msg_param_type's values
 * params   - the parameter payload
 *
 * This function works by setting up a temporary parameter context within the
 * model-net implementation (currently implemented as a set of translation-unit
 * globals). Upon a subsequent model_net_*event* call, the context is consumed
 * and reset to an unused state.
391
 *
392 393 394 395 396 397 398 399 400 401
 * NOTE: this call MUST be placed in the same calling context as the subsequent
 * model_net_*event* call. Otherwise, the parameters are not guaranteed to work
 * on the intended event, and may possibly be consumed by another, unrelated
 * event.
 */
void model_net_set_msg_param(
        enum msg_param_type type,
        int sub_type,
        const void * params);

402 403
/* returns pointer to LP information for simplenet module */
const tw_lptype* model_net_get_lp_type(int net_id);
404
const st_model_types* model_net_get_model_stat_type(int net_id);
405

406
DEPRECATED
407
uint64_t model_net_get_packet_size(int net_id);
408

409 410 411
/* used for reporting overall network statistics for e.g. average latency ,
 * maximum latency, total number of packets finished during the entire
 * simulation etc. */
412
void model_net_report_stats(int net_id);
413

414
/* writing model-net statistics on a per LP basis */
415 416
void model_net_write_stats(tw_lpid lpid, mn_stats* stat);

417
/* printing model-net statistics on a per LP basis */
418 419 420
void model_net_print_stats(tw_lpid lpid, mn_stats mn_stats_array[]);

/* find model-net statistics */
421
mn_stats* model_net_find_stats(char const * category, mn_stats mn_stats_array[]);
Jonathan Jenkins's avatar
Jonathan Jenkins committed
422

423 424 425 426 427
#ifdef ENABLE_CORTEX
/* structure that gives access to the topology functions */
extern cortex_topology model_net_topology;
#endif

Jonathan Jenkins's avatar
Jonathan Jenkins committed
428 429 430 431
#ifdef __cplusplus
}
#endif

432 433 434 435 436 437 438 439 440 441
#endif /* MODELNET_H */

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */