simplenet-upd.c 18.2 KB
Newer Older
1
/*
Philip Carns's avatar
Philip Carns committed
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
Philip Carns's avatar
Philip Carns committed
4
 *
5 6 7 8 9 10
 */

#include <string.h>
#include <assert.h>
#include <ross.h>

11 12
#include "codes/lp-io.h"
#include "codes/jenkins-hash.h"
13
#include "codes/model-net-method.h"
14
#include "codes/model-net.h"
15
#include "codes/model-net-lp.h"
16 17
#include "codes/codes_mapping.h"
#include "codes/codes.h"
18
#include "codes/net/simplenet-upd.h"
19 20 21 22

#define CATEGORY_NAME_MAX 16
#define CATEGORY_MAX 12

23 24 25
#define LP_CONFIG_NM (model_net_lp_config_names[SIMPLENET])
#define LP_METHOD_NM (model_net_method_names[SIMPLENET])

26 27 28 29 30 31 32 33
/* structs for initializing a network/ specifying network parameters */
struct simplenet_param
{
  double net_startup_ns; /*simplenet startup cost*/
  double net_bw_mbps; /*Link bandwidth per byte*/
};
typedef struct simplenet_param simplenet_param;

34 35 36 37 38 39 40 41
/*Define simplenet data types and structs*/
typedef struct sn_state sn_state;

struct sn_state
{
    /* next idle times for network card, both inbound and outbound */
    tw_stime net_send_next_idle;
    tw_stime net_recv_next_idle;
42 43
    const char * anno;
    simplenet_param params;
44
    struct mn_stats sn_stats_array[CATEGORY_MAX];
45 46
};

47
/* annotation-specific parameters (unannotated entry occurs at the
48 49 50 51
 * last index) */
static uint64_t                  num_params = 0;
static simplenet_param         * all_params = NULL;
static const config_anno_map_t * anno_map   = NULL;
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66

static int sn_magic = 0;

/* returns a pointer to the lptype struct to use for simplenet LPs */
static const tw_lptype* sn_get_lp_type(void);

/* retrieve the size of the portion of the event struct that is consumed by
 * the simplenet module.  The caller should add this value to the size of
 * its own event structure to get the maximum total size of a message.
 */
static int sn_get_msg_sz(void);

/* Returns the simplenet magic number */
static int sn_get_magic();

67
#if SIMPLENET_DEBUG
68
static void print_msg(sn_message *m);
69
#endif
70

71 72 73 74 75 76
/* collective network calls */
static void simple_net_collective();

/* collective network calls-- rc */
static void simple_net_collective_rc();

77 78 79 80
/* Modelnet interface events */
/* sets up the simplenet parameters through modelnet interface */
static void sn_configure();

81 82 83 84 85 86 87 88 89 90 91 92 93
/* allocate a new event that will pass through simplenet to arriave at its
 * destination:
 *
 * - category: category name to associate with this communication
 * - final_dest_gid: the LP that the message should be delivered to.
 * - event_size_bytes: size of event msg that will be delivered to
 * final_dest_gid.
 * - local_event_size_byte: size of event message that will delivered to
 *   local LP upon local send comletion (set to 0 if not used)
 * - net_msg_size_bytes: size of simulated network message in bytes.
 * - sender: LP calling this function.
 */
/* Issues a simplenet packet event call */
94
static tw_stime simplenet_packet_event(
95 96 97 98 99 100 101 102 103
        model_net_request const * req,
        uint64_t message_offset,
        uint64_t packet_size,
        tw_stime offset,
        mn_sched_params const * sched_params,
        void const * remote_event,
        void const * self_event,
        tw_lp *sender,
        int is_last_pckt);
104

105 106 107
static void simplenet_packet_event_rc(tw_lp *sender);

static void simplenet_packet_event_rc(tw_lp *sender);
108 109 110

static void sn_report_stats();

111 112 113
/* data structure for model-net statistics */
struct model_net_method simplenet_method =
{
114
    .mn_configure = sn_configure,
115
    .mn_register = NULL,
116 117
    .model_net_method_packet_event = simplenet_packet_event,
    .model_net_method_packet_event_rc = simplenet_packet_event_rc,
118 119
    .model_net_method_recv_msg_event = NULL,
    .model_net_method_recv_msg_event_rc = NULL,
120 121
    .mn_get_lp_type = sn_get_lp_type,
    .mn_get_msg_sz = sn_get_msg_sz,
122
    .mn_report_stats = sn_report_stats,
123 124 125 126 127
    .mn_collective_call = simple_net_collective,
    .mn_collective_call_rc = simple_net_collective_rc,
    .mn_sample_fn = NULL,
    .mn_sample_rc_fn = NULL,
    .mn_sample_init_fn = NULL,
128 129 130
    .mn_sample_fini_fn = NULL,
    .mn_model_stat_register = NULL, // for ROSS instrumentation
    .mn_get_model_stat_types = NULL // for ROSS instrumentation
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
};

static void sn_init(
    sn_state * ns,
    tw_lp * lp);
static void sn_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void sn_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void sn_finalize(
    sn_state * ns,
    tw_lp * lp);

tw_lptype sn_lp = {
151 152 153 154
    (init_f) sn_init,
    (pre_run_f) NULL,
    (event_f) sn_event,
    (revent_f) sn_rev_event,
155
    (commit_f) NULL,
156 157 158
    (final_f) sn_finalize,
    (map_f) codes_mapping,
    sizeof(sn_state),
159 160
};

161
static tw_stime rate_to_ns(uint64_t bytes, double MB_p_s);
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
static void handle_msg_ready_rev_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_ready_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_start_rev_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_start_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp);

/* returns pointer to LP information for simplenet module */
static const tw_lptype* sn_get_lp_type()
{
    return(&sn_lp);
}

/* returns number of bytes that the simplenet module will consume in event
 * messages
 */
static int sn_get_msg_sz(void)
{
    return(sizeof(sn_message));
}
192 193 194 195 196 197 198 199 200 201 202 203 204
/* collective network calls */
static void simple_net_collective()
{
/* collectives not supported */
    return;
}

/* collective network call -- rc*/
static void simple_net_collective_rc()
{
/* collectives not supported */
   return;
}
205

206 207 208 209 210 211
/* report network statistics */
static void sn_report_stats()
{
   /* TODO: Do we have some simplenet statistics to report like we have for torus and dragonfly? */
   return;
}
212 213 214 215
static void sn_init(
    sn_state * ns,
    tw_lp * lp)
{
216
    uint32_t h1 = 0, h2 = 0;
217 218 219 220 221 222
    memset(ns, 0, sizeof(*ns));

    /* all devices are idle to begin with */
    ns->net_send_next_idle = tw_now(lp);
    ns->net_recv_next_idle = tw_now(lp);

223 224 225 226 227 228 229 230
    ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid);
    if (ns->anno == NULL)
        ns->params = all_params[num_params-1];
    else{
        int id = configuration_get_annotation_index(ns->anno, anno_map);
        ns->params = all_params[id];
    }

231
    bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
232 233 234 235 236 237 238 239 240 241 242 243
    sn_magic = h1+h2;
    /* printf("\n sn_magic %d ", sn_magic); */

    return;
}

static void sn_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
244
    (void)b; // bitflags aren't used in simplenet
245
    assert(m->magic == sn_magic);
246 247 248

    switch (m->event_type)
    {
249
        case SN_MSG_START:
250
            handle_msg_start_event(ns, m, lp);
251
            break;
252
        case SN_MSG_READY:
253
            handle_msg_ready_event(ns, m, lp);
254 255 256 257 258 259 260 261 262 263 264 265 266
            break;
        default:
            assert(0);
            break;
    }
}

static void sn_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
267
    (void)b;
268
    assert(m->magic == sn_magic);
269 270 271

    switch (m->event_type)
    {
272
        case SN_MSG_START:
273
            handle_msg_start_rev_event(ns, m, lp);
274
            break;
275
        case SN_MSG_READY:
276
            handle_msg_ready_rev_event(ns, m, lp);
277 278 279 280 281 282 283 284 285 286 287 288 289
            break;
        default:
            assert(0);
            break;
    }

    return;
}

static void sn_finalize(
    sn_state * ns,
    tw_lp * lp)
{
290
    model_net_print_stats(lp->gid, &ns->sn_stats_array[0]);
291 292 293 294 295 296 297 298 299
    return;
}

int sn_get_magic()
{
  return sn_magic;
}

/* convert MiB/s and bytes to ns */
300
static tw_stime rate_to_ns(uint64_t bytes, double MB_p_s)
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
{
    tw_stime time;

    /* bytes to MB */
    time = ((double)bytes)/(1024.0*1024.0);
    /* MB to s */
    time = time / MB_p_s;
    /* s to ns */
    time = time * 1000.0 * 1000.0 * 1000.0;

    return(time);
}

/* reverse computation for msg ready event */
static void handle_msg_ready_rev_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp)
{
320
    struct mn_stats* stat;
321 322

    ns->net_recv_next_idle = m->net_recv_next_idle_saved;
323

324
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
325 326
    stat->recv_count--;
    stat->recv_bytes -= m->net_msg_size_bytes;
327
    stat->recv_time = m->recv_time_saved;
328

329
    if (m->event_size_bytes && m->is_pull){
330
        model_net_event_rc2(lp, &m->event_rc);
331 332
    }

333 334 335 336 337 338 339 340 341 342 343 344
    return;
}

/* handler for msg ready event.  This indicates that a message is available
 * to recv, but we haven't checked to see if the recv queue is available yet
 */
static void handle_msg_ready_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp)
{
    tw_stime recv_queue_time = 0;
345
    struct mn_stats* stat;
346 347 348

    //printf("handle_msg_ready_event(), lp %llu.\n", (unsigned long long)lp->gid);
    /* add statistics */
349
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
350 351
    stat->recv_count++;
    stat->recv_bytes += m->net_msg_size_bytes;
352
    m->recv_time_saved = stat->recv_time;
353 354
    stat->recv_time += rate_to_ns(m->net_msg_size_bytes,
            ns->params.net_bw_mbps);
355 356 357 358 359 360 361

    /* are we available to recv the msg? */
    /* were we available when the transmission was started? */
    if(ns->net_recv_next_idle > tw_now(lp))
        recv_queue_time += ns->net_recv_next_idle - tw_now(lp);

    /* calculate transfer time based on msg size and bandwidth */
362 363
    recv_queue_time += rate_to_ns(m->net_msg_size_bytes,
            ns->params.net_bw_mbps);
364 365 366 367 368 369 370 371

    /* bump up input queue idle time accordingly */
    m->net_recv_next_idle_saved = ns->net_recv_next_idle;
    ns->net_recv_next_idle = recv_queue_time + tw_now(lp);

    /* copy only the part of the message used by higher level */
    if(m->event_size_bytes)
    {
372 373 374
        //char *tmp_ptr = (char*)m;
        //tmp_ptr += sn_get_msg_sz();
        void *tmp_ptr = model_net_method_get_edata(SIMPLENET, m);
375
      /* schedule event to final destination for when the recv is complete */
376
//      printf("\n Remote message to LP %d ", m->final_dest_gid);
377
        if (m->is_pull){
378 379 380 381 382 383
            /* call the model-net event, using direct contexts for mapping (we
             * know all involved LPs */
            struct codes_mctx mc_dst =
                codes_mctx_set_global_direct(m->src_mn_lp);
            struct codes_mctx mc_src =
                codes_mctx_set_global_direct(lp->gid);
384
            int net_id = model_net_get_id(LP_METHOD_NM);
385 386
            m->event_rc = model_net_event_mctx(net_id, &mc_src, &mc_dst,
                    m->category, m->src_gid, m->pull_size, recv_queue_time,
387
                    m->event_size_bytes, tmp_ptr, 0, NULL, lp);
388 389
        }
        else{
390 391
            tw_event * e_new = tw_event_new(m->final_dest_gid, recv_queue_time, lp);
            void * m_new = tw_event_data(e_new);
392 393 394
            memcpy(m_new, tmp_ptr, m->event_size_bytes);
            tw_event_send(e_new);
        }
395 396 397 398 399 400 401 402 403 404 405 406 407
    }

    return;
}

/* reverse computation for msg start event */
static void handle_msg_start_rev_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp)
{
    ns->net_send_next_idle = m->net_send_next_idle_saved;

408 409
    codes_local_latency_reverse(lp);

410 411 412 413 414
    if(m->local_event_size_bytes > 0)
    {
        codes_local_latency_reverse(lp);
    }

415 416
    mn_stats* stat;
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
417 418
    stat->send_count--;
    stat->send_bytes -= m->net_msg_size_bytes;
419
    stat->send_time = m->send_time_saved;
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434

    return;
}

/* handler for msg start event; this indicates that the caller is trying to
 * transmit a message through this NIC
 */
static void handle_msg_start_event(
    sn_state * ns,
    sn_message * m,
    tw_lp * lp)
{
    tw_event *e_new;
    sn_message *m_new;
    tw_stime send_queue_time = 0;
435
    mn_stats* stat;
436 437
    int total_event_size;

438 439
    total_event_size = model_net_get_msg_sz(SIMPLENET) + m->event_size_bytes +
        m->local_event_size_bytes;
440 441 442

    //printf("handle_msg_start_event(), lp %llu.\n", (unsigned long long)lp->gid);
    /* add statistics */
443
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
444 445
    stat->send_count++;
    stat->send_bytes += m->net_msg_size_bytes;
446
    m->send_time_saved = stat->send_time;
447 448
    stat->send_time += (ns->params.net_startup_ns + rate_to_ns(m->net_msg_size_bytes,
                ns->params.net_bw_mbps));
449 450
    if(stat->max_event_size < total_event_size)
        stat->max_event_size = total_event_size;
451 452

    /* calculate send time stamp */
453
    send_queue_time = ns->params.net_startup_ns; /* net msg startup cost */
454 455 456 457 458
    /* bump up time if the NIC send queue isn't idle right now */
    if(ns->net_send_next_idle > tw_now(lp))
        send_queue_time += ns->net_send_next_idle - tw_now(lp);

    /* move the next idle time ahead to after this transmission is
459 460
     * _complete_ from the sender's perspective
     */
461 462
    m->net_send_next_idle_saved = ns->net_send_next_idle;
    ns->net_send_next_idle = send_queue_time + tw_now(lp) +
463
        rate_to_ns(m->net_msg_size_bytes, ns->params.net_bw_mbps);
464

465
    void *m_data;
466
    e_new = model_net_method_event_new(m->dest_mn_lp, send_queue_time, lp,
467
            SIMPLENET, (void**)&m_new, &m_data);
468 469 470 471

    /* copy entire previous message over, including payload from user of
     * this module
     */
472 473 474 475 476 477
    //memcpy(m_new, m, m->event_size_bytes + model_net_get_msg_sz(SIMPLENET));
    memcpy(m_new, m, sizeof(sn_message));
    if (m->event_size_bytes){
        memcpy(m_data, model_net_method_get_edata(SIMPLENET, m),
                m->event_size_bytes);
    }
478

479
    m_new->event_type = SN_MSG_READY;
480

481 482
    //print_base_from(SIMPLENET, m_new);
    //print_msg(m_new);
483 484
    tw_event_send(e_new);

485 486 487
    // now that message is sent, issue an "idle" event to tell the scheduler
    // when I'm next available
    model_net_method_idle_event(codes_local_latency(lp) +
488
            ns->net_send_next_idle - tw_now(lp), 0, lp);
489

490 491 492 493 494
    /* if there is a local event to handle, then create an event for it as
     * well
     */
    if(m->local_event_size_bytes > 0)
    {
495
        //char* local_event;
496

497
        e_new = tw_event_new(m->src_gid, send_queue_time+codes_local_latency(lp), lp);
498 499
        m_new = tw_event_data(e_new);

500 501 502 503
        void * m_loc = (char*) model_net_method_get_edata(SIMPLENET, m) +
            m->event_size_bytes;

         //local_event = (char*)m;
504
         //local_event += model_net_get_msg_sz(SIMPLENET) + m->event_size_bytes;
505 506 507
        /* copy just the local event data over (which is past the remote event
         * in memory) */
        memcpy(m_new, m_loc, m->local_event_size_bytes);
508 509 510 511 512 513 514
        tw_event_send(e_new);
    }
    return;
}

/* Model-net function calls */

515
/*This method will serve as an intermediate layer between simplenet and modelnet.
516
 * It takes the packets from modelnet layer and calls underlying simplenet methods*/
517
static tw_stime simplenet_packet_event(
518 519 520 521 522 523 524 525 526
        model_net_request const * req,
        uint64_t message_offset,
        uint64_t packet_size,
        tw_stime offset,
        mn_sched_params const * sched_params,
        void const * remote_event,
        void const * self_event,
        tw_lp *sender,
        int is_last_pckt)
527
{
528 529 530
     (void)message_offset; // unused...
     (void)sched_params; // unused...

531 532 533 534 535 536
     tw_event * e_new;
     tw_stime xfer_to_nic_time;
     sn_message * msg;
     char* tmp_ptr;

     xfer_to_nic_time = codes_local_latency(sender);
537 538 539
     // this is a self message
     e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
             sender, SIMPLENET, (void**)&msg, (void**)&tmp_ptr);
540 541
     strcpy(msg->category, req->category);
     msg->src_gid = req->src_lp;
542
     msg->src_mn_lp = sender->gid;
543 544
     msg->final_dest_gid = req->final_dest_lp;
     msg->dest_mn_lp = req->dest_mn_lp;
545 546 547 548
     msg->magic = sn_get_magic();
     msg->net_msg_size_bytes = packet_size;
     msg->event_size_bytes = 0;
     msg->local_event_size_bytes = 0;
549
     msg->event_type = SN_MSG_START;
550 551
     msg->is_pull = req->is_pull;
     msg->pull_size = req->pull_size;
552

553
     /*Fill in simplenet information*/
554 555
     if(is_last_pckt) /* Its the last packet so pass in remote event information*/
      {
556
       if(req->remote_event_size)
557
	 {
558 559 560
           msg->event_size_bytes = req->remote_event_size;
           memcpy(tmp_ptr, remote_event, req->remote_event_size);
           tmp_ptr += req->remote_event_size;
561
	 }
562
       if(req->self_event_size)
563
       {
564 565 566
	   msg->local_event_size_bytes = req->self_event_size;
	   memcpy(tmp_ptr, self_event, req->self_event_size);
	   tmp_ptr += req->self_event_size;
567 568 569
       }
      }
     tw_event_send(e_new);
570
     return xfer_to_nic_time;
571 572
}

573
static void sn_configure()
574
{
575 576 577 578
    anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
    assert(anno_map);
    num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
    all_params = malloc(num_params * sizeof(*all_params));
579
    for (int i = 0; i < anno_map->num_annos; i++){
580
        const char * anno = anno_map->annotations[i].ptr;
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
        int rc;
        rc = configuration_get_value_double(&config, "PARAMS",
                "net_startup_ns", anno, &all_params[i].net_startup_ns);
        if (rc != 0){
            tw_error(TW_LOC,
                    "simplenet: unable to read PARAMS:net_startup_ns@%s",
                    anno);
        }
        rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps",
                anno, &all_params[i].net_bw_mbps);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps@%s",
                    anno);
        }
    }
    if (anno_map->has_unanno_lp > 0){
        int rc;
        rc = configuration_get_value_double(&config, "PARAMS",
                "net_startup_ns", NULL,
                &all_params[num_params-1].net_startup_ns);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_startup_ns");
        }
        rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps",
                NULL, &all_params[num_params-1].net_bw_mbps);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps");
        }
    }
610 611 612 613 614 615 616 617
}

static void simplenet_packet_event_rc(tw_lp *sender)
{
    codes_local_latency_reverse(sender);
    return;
}

618
#if SIMPLENET_DEBUG
619 620 621 622 623
void print_msg(sn_message *m){
    printf(" sn:\n  type:%d, magic:%d, src:%lu, dest:%lu, esize:%d, lsize:%d\n",
            m->event_type, m->magic, m->src_gid, m->final_dest_gid,
            m->event_size_bytes, m->local_event_size_bytes);
}
624
#endif
625

626

627 628 629 630 631 632 633 634
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */