simplenet-upd.c 19.2 KB
Newer Older
1
/*
Philip Carns's avatar
Philip Carns committed
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
Philip Carns's avatar
Philip Carns committed
4
 *
5
6
7
8
9
10
 */

#include <string.h>
#include <assert.h>
#include <ross.h>

11
12
#include "codes/lp-io.h"
#include "codes/jenkins-hash.h"
13
#include "codes/model-net-method.h"
14
#include "codes/model-net.h"
15
#include "codes/model-net-lp.h"
16
17
#include "codes/codes_mapping.h"
#include "codes/codes.h"
18
#include "codes/net/simplenet-upd.h"
19
20
21
22

#define CATEGORY_NAME_MAX 16
#define CATEGORY_MAX 12

23
24
25
#define LP_CONFIG_NM (model_net_lp_config_names[SIMPLENET])
#define LP_METHOD_NM (model_net_method_names[SIMPLENET])

26
27
28
29
30
31
32
33
/* structs for initializing a network/ specifying network parameters */
struct simplenet_param
{
  double net_startup_ns; /*simplenet startup cost*/
  double net_bw_mbps; /*Link bandwidth per byte*/
};
typedef struct simplenet_param simplenet_param;

34
35
36
37
38
39
40
41
/*Define simplenet data types and structs*/
typedef struct sn_state sn_state;

struct sn_state
{
    /* next idle times for network card, both inbound and outbound */
    tw_stime net_send_next_idle;
    tw_stime net_recv_next_idle;
42
43
    const char * anno;
    simplenet_param params;
44
    struct mn_stats sn_stats_array[CATEGORY_MAX];
45
46
};

47
48
49
50
51
/* annotation-specific parameters (unannotated entry occurs at the 
 * last index) */
static uint64_t                  num_params = 0;
static simplenet_param         * all_params = NULL;
static const config_anno_map_t * anno_map   = NULL;
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

static int sn_magic = 0;

/* returns a pointer to the lptype struct to use for simplenet LPs */
static const tw_lptype* sn_get_lp_type(void);

/* retrieve the size of the portion of the event struct that is consumed by
 * the simplenet module.  The caller should add this value to the size of
 * its own event structure to get the maximum total size of a message.
 */
static int sn_get_msg_sz(void);

/* Returns the simplenet magic number */
static int sn_get_magic();

67
#if SIMPLENET_DEBUG
68
static void print_msg(sn_message *m);
69
#endif
70

71
72
73
74
75
76
/* collective network calls */
static void simple_net_collective();

/* collective network calls-- rc */
static void simple_net_collective_rc();

77
78
79
80
81
82
83
84
85
86
87
88
89
90
/* allocate a new event that will pass through simplenet to arriave at its
 * destination:
 *
 * - category: category name to associate with this communication
 * - final_dest_gid: the LP that the message should be delivered to.
 * - event_size_bytes: size of event msg that will be delivered to
 * final_dest_gid.
 * - local_event_size_byte: size of event message that will delivered to
 *   local LP upon local send comletion (set to 0 if not used)
 * - net_msg_size_bytes: size of simulated network message in bytes.
 * - sender: LP calling this function.
 */
/* Modelnet interface events */
/* sets up the simplenet parameters through modelnet interface */
91
static void sn_configure();
92
93

/* Issues a simplenet packet event call */
94
static tw_stime simplenet_packet_event(
95
96
     char* category, 
     tw_lpid final_dest_lp, 
97
     uint64_t packet_size, 
98
99
     int is_pull,
     uint64_t pull_size, /* only used when is_pull==1 */
100
     tw_stime offset,
101
     const mn_sched_params *sched_params,
102
103
104
105
     int remote_event_size, 
     const void* remote_event, 
     int self_event_size,
     const void* self_event,
106
     tw_lpid src_lp,
107
108
109
110
111
     tw_lp *sender,
     int is_last_pckt);
static void simplenet_packet_event_rc(tw_lp *sender);

static void simplenet_packet_event_rc(tw_lp *sender);
112
113
114

static void sn_report_stats();

115
116
117
118
static tw_lpid sn_find_local_device(
        const char * annotation, 
        int          ignore_annotations,
        tw_lp      * sender);
119

120
121
122
/* data structure for model-net statistics */
struct model_net_method simplenet_method =
{
123
    .mn_configure = sn_configure,
124
125
    .model_net_method_packet_event = simplenet_packet_event,
    .model_net_method_packet_event_rc = simplenet_packet_event_rc,
126
127
    .model_net_method_recv_msg_event = NULL,
    .model_net_method_recv_msg_event_rc = NULL,
128
129
    .mn_get_lp_type = sn_get_lp_type,
    .mn_get_msg_sz = sn_get_msg_sz,
130
    .mn_report_stats = sn_report_stats,
131
    .model_net_method_find_local_device = sn_find_local_device,
132
133
    .mn_collective_call = simple_net_collective,	
    .mn_collective_call_rc = simple_net_collective_rc
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
};

static void sn_init(
    sn_state * ns,
    tw_lp * lp);
static void sn_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void sn_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void sn_finalize(
    sn_state * ns,
    tw_lp * lp);

tw_lptype sn_lp = {
154
155
156
157
158
159
160
    (init_f) sn_init,
    (pre_run_f) NULL,
    (event_f) sn_event,
    (revent_f) sn_rev_event,
    (final_f) sn_finalize,
    (map_f) codes_mapping,
    sizeof(sn_state),
161
162
};

163
static tw_stime rate_to_ns(uint64_t bytes, double MB_p_s);
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
static void handle_msg_ready_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_ready_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_start_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_start_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);

/* returns pointer to LP information for simplenet module */
static const tw_lptype* sn_get_lp_type()
{
    return(&sn_lp);
}

/* returns number of bytes that the simplenet module will consume in event
 * messages
 */
static int sn_get_msg_sz(void)
{
    return(sizeof(sn_message));
}
198
199
200
201
202
203
204
205
206
207
208
209
210
/* collective network calls */
static void simple_net_collective()
{
/* collectives not supported */
    return;
}

/* collective network call -- rc*/
static void simple_net_collective_rc()
{
/* collectives not supported */
   return;
}
211

212
213
214
215
216
217
/* report network statistics */
static void sn_report_stats()
{
   /* TODO: Do we have some simplenet statistics to report like we have for torus and dragonfly? */
   return;
}
218
219
220
221
static void sn_init(
    sn_state * ns,
    tw_lp * lp)
{
222
    uint32_t h1 = 0, h2 = 0;
223
224
225
226
227
228
    memset(ns, 0, sizeof(*ns));

    /* all devices are idle to begin with */
    ns->net_send_next_idle = tw_now(lp);
    ns->net_recv_next_idle = tw_now(lp);

229
230
231
232
233
234
235
236
    ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid);
    if (ns->anno == NULL)
        ns->params = all_params[num_params-1];
    else{
        int id = configuration_get_annotation_index(ns->anno, anno_map);
        ns->params = all_params[id];
    }

237
    bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
238
239
240
241
242
243
244
245
246
247
248
249
    sn_magic = h1+h2;
    /* printf("\n sn_magic %d ", sn_magic); */

    return;
}

static void sn_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
250
    assert(m->magic == sn_magic);
251
252
253

    switch (m->event_type)
    {
254
        case SN_MSG_START:
255
256
            handle_msg_start_event(ns, b, m, lp);
            break;
257
        case SN_MSG_READY:
258
259
260
261
262
263
264
265
266
267
268
269
270
271
            handle_msg_ready_event(ns, b, m, lp);
            break;
        default:
            assert(0);
            break;
    }
}

static void sn_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
272
    assert(m->magic == sn_magic);
273
274
275

    switch (m->event_type)
    {
276
        case SN_MSG_START:
277
278
            handle_msg_start_rev_event(ns, b, m, lp);
            break;
279
        case SN_MSG_READY:
280
281
282
283
284
285
286
287
288
289
290
291
292
293
            handle_msg_ready_rev_event(ns, b, m, lp);
            break;
        default:
            assert(0);
            break;
    }

    return;
}

static void sn_finalize(
    sn_state * ns,
    tw_lp * lp)
{
294
    model_net_print_stats(lp->gid, &ns->sn_stats_array[0]);
295
296
297
298
299
300
301
302
303
    return;
}

int sn_get_magic()
{
  return sn_magic;
}

/* convert MiB/s and bytes to ns */
304
static tw_stime rate_to_ns(uint64_t bytes, double MB_p_s)
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
{
    tw_stime time;

    /* bytes to MB */
    time = ((double)bytes)/(1024.0*1024.0);
    /* MB to s */
    time = time / MB_p_s;
    /* s to ns */
    time = time * 1000.0 * 1000.0 * 1000.0;

    return(time);
}

/* reverse computation for msg ready event */
static void handle_msg_ready_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
325
    struct mn_stats* stat;
326
327
328

    ns->net_recv_next_idle = m->net_recv_next_idle_saved;
    
329
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
330
331
    stat->recv_count--;
    stat->recv_bytes -= m->net_msg_size_bytes;
332
    stat->recv_time = m->recv_time_saved;
333

334
    if (m->event_size_bytes && m->is_pull){
335
        int net_id = model_net_get_id(LP_METHOD_NM);
336
337
338
        model_net_event_rc(net_id, lp, m->pull_size);
    }

339
340
341
342
343
344
345
346
347
348
349
350
351
    return;
}

/* handler for msg ready event.  This indicates that a message is available
 * to recv, but we haven't checked to see if the recv queue is available yet
 */
static void handle_msg_ready_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
    tw_stime recv_queue_time = 0;
352
    struct mn_stats* stat;
353
354
355

    //printf("handle_msg_ready_event(), lp %llu.\n", (unsigned long long)lp->gid);
    /* add statistics */
356
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
357
358
    stat->recv_count++;
    stat->recv_bytes += m->net_msg_size_bytes;
359
    m->recv_time_saved = stat->recv_time;
360
361
    stat->recv_time += rate_to_ns(m->net_msg_size_bytes,
            ns->params.net_bw_mbps);
362
363
364
365
366
367
368

    /* are we available to recv the msg? */
    /* were we available when the transmission was started? */
    if(ns->net_recv_next_idle > tw_now(lp))
        recv_queue_time += ns->net_recv_next_idle - tw_now(lp);

    /* calculate transfer time based on msg size and bandwidth */
369
370
    recv_queue_time += rate_to_ns(m->net_msg_size_bytes,
            ns->params.net_bw_mbps);
371
372
373
374
375
376
377
378

    /* bump up input queue idle time accordingly */
    m->net_recv_next_idle_saved = ns->net_recv_next_idle;
    ns->net_recv_next_idle = recv_queue_time + tw_now(lp);

    /* copy only the part of the message used by higher level */
    if(m->event_size_bytes)
    {
379
380
381
        //char *tmp_ptr = (char*)m;
        //tmp_ptr += sn_get_msg_sz();
        void *tmp_ptr = model_net_method_get_edata(SIMPLENET, m);
382
383
      /* schedule event to final destination for when the recv is complete */
//      printf("\n Remote message to LP %d ", m->final_dest_gid); 
384
385
        if (m->is_pull){
            /* call the model-net event */
386
            int net_id = model_net_get_id(LP_METHOD_NM);
387
388
389
390
391
            model_net_event(net_id, m->category, m->src_gid, m->pull_size,
                    recv_queue_time, m->event_size_bytes, tmp_ptr, 0, NULL,
                    lp);
        }
        else{
392
393
            tw_event * e_new = tw_event_new(m->final_dest_gid, recv_queue_time, lp);
            void * m_new = tw_event_data(e_new);
394
395
396
            memcpy(m_new, tmp_ptr, m->event_size_bytes);
            tw_event_send(e_new);
        }
397
398
399
400
401
402
403
404
405
406
407
408
409
410
    }

    return;
}

/* reverse computation for msg start event */
static void handle_msg_start_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
    ns->net_send_next_idle = m->net_send_next_idle_saved;

411
412
    codes_local_latency_reverse(lp);

413
414
415
416
417
    if(m->local_event_size_bytes > 0)
    {
        codes_local_latency_reverse(lp);
    }

418
419
    mn_stats* stat;
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
420
421
    stat->send_count--;
    stat->send_bytes -= m->net_msg_size_bytes;
422
    stat->send_time = m->send_time_saved;
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438

    return;
}

/* handler for msg start event; this indicates that the caller is trying to
 * transmit a message through this NIC
 */
static void handle_msg_start_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
    tw_event *e_new;
    sn_message *m_new;
    tw_stime send_queue_time = 0;
439
    mn_stats* stat;
440
    int mapping_rep_id, mapping_offset, dummy;
441
    tw_lpid dest_id;
442
    char lp_group_name[MAX_NAME_LENGTH];
443
444
    int total_event_size;

445
446
    total_event_size = model_net_get_msg_sz(SIMPLENET) + m->event_size_bytes +
        m->local_event_size_bytes;
447
448
449

    //printf("handle_msg_start_event(), lp %llu.\n", (unsigned long long)lp->gid);
    /* add statistics */
450
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
451
452
    stat->send_count++;
    stat->send_bytes += m->net_msg_size_bytes;
453
    m->send_time_saved = stat->send_time;
454
455
    stat->send_time += (ns->params.net_startup_ns + rate_to_ns(m->net_msg_size_bytes,
                ns->params.net_bw_mbps));
456
457
    if(stat->max_event_size < total_event_size)
        stat->max_event_size = total_event_size;
458
459

    /* calculate send time stamp */
460
    send_queue_time = ns->params.net_startup_ns; /* net msg startup cost */
461
462
463
464
465
466
467
468
469
    /* bump up time if the NIC send queue isn't idle right now */
    if(ns->net_send_next_idle > tw_now(lp))
        send_queue_time += ns->net_send_next_idle - tw_now(lp);

    /* move the next idle time ahead to after this transmission is
     * _complete_ from the sender's perspective 
     */ 
    m->net_send_next_idle_saved = ns->net_send_next_idle;
    ns->net_send_next_idle = send_queue_time + tw_now(lp) +
470
        rate_to_ns(m->net_msg_size_bytes, ns->params.net_bw_mbps);
471
472
473


    /* create new event to send msg to receiving NIC */
474
475
476
    // TODO: don't ignore annotations
    codes_mapping_get_lp_info(m->final_dest_gid, lp_group_name, &dummy, NULL,
            &dummy, NULL, &mapping_rep_id, &mapping_offset);
477
    codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, ns->anno, 0,
478
            mapping_rep_id, mapping_offset, &dest_id); 
479
480

//    printf("\n msg start sending to %d ", dest_id);
481
482
483
484
485
    void *m_data;
    //e_new = tw_event_new(dest_id, send_queue_time, lp);
    //m_new = tw_event_data(e_new);
    e_new = model_net_method_event_new(dest_id, send_queue_time, lp,
            SIMPLENET, (void**)&m_new, &m_data);
486
487
488
489

    /* copy entire previous message over, including payload from user of
     * this module
     */
490
491
492
493
494
495
496
497
    //memcpy(m_new, m, m->event_size_bytes + model_net_get_msg_sz(SIMPLENET));
    memcpy(m_new, m, sizeof(sn_message));
    if (m->event_size_bytes){
        memcpy(m_data, model_net_method_get_edata(SIMPLENET, m),
                m->event_size_bytes);
    }
    
    m_new->event_type = SN_MSG_READY;
498
    
499
500
    //print_base_from(SIMPLENET, m_new);
    //print_msg(m_new);
501
502
    tw_event_send(e_new);

503
504
505
    // now that message is sent, issue an "idle" event to tell the scheduler
    // when I'm next available
    model_net_method_idle_event(codes_local_latency(lp) +
506
            ns->net_send_next_idle - tw_now(lp), 0, lp);
507

508
509
510
511
512
    /* if there is a local event to handle, then create an event for it as
     * well
     */
    if(m->local_event_size_bytes > 0)
    {
513
        //char* local_event;
514

515
        e_new = tw_event_new(m->src_gid, send_queue_time+codes_local_latency(lp), lp);
516
517
        m_new = tw_event_data(e_new);

518
519
520
521
522
523
524
525
        void * m_loc = (char*) model_net_method_get_edata(SIMPLENET, m) +
            m->event_size_bytes;

         //local_event = (char*)m;
         //local_event += model_net_get_msg_sz(SIMPLENET) + m->event_size_bytes;         	 
        /* copy just the local event data over (which is past the remote event
         * in memory) */
        memcpy(m_new, m_loc, m->local_event_size_bytes);
526
527
528
529
530
531
532
533
534
        tw_event_send(e_new);
    }
    return;
}

/* Model-net function calls */

/*This method will serve as an intermediate layer between simplenet and modelnet. 
 * It takes the packets from modelnet layer and calls underlying simplenet methods*/
535
static tw_stime simplenet_packet_event(
536
537
		char* category,
		tw_lpid final_dest_lp,
538
		uint64_t packet_size,
539
540
                int is_pull,
                uint64_t pull_size, /* only used when is_pull == 1 */
541
                tw_stime offset,
542
                const mn_sched_params *sched_params,
543
544
545
546
		int remote_event_size,
		const void* remote_event,
		int self_event_size,
		const void* self_event,
547
                tw_lpid src_lp,
548
549
550
551
552
553
554
555
556
		tw_lp *sender,
		int is_last_pckt)
{
     tw_event * e_new;
     tw_stime xfer_to_nic_time;
     sn_message * msg;
     char* tmp_ptr;

     xfer_to_nic_time = codes_local_latency(sender);
557
558
559
     // this is a self message
     e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
             sender, SIMPLENET, (void**)&msg, (void**)&tmp_ptr);
560
561
     strcpy(msg->category, category);
     msg->final_dest_gid = final_dest_lp;
562
     msg->src_gid = src_lp;
563
564
565
566
     msg->magic = sn_get_magic();
     msg->net_msg_size_bytes = packet_size;
     msg->event_size_bytes = 0;
     msg->local_event_size_bytes = 0;
567
     msg->event_type = SN_MSG_START;
568
569
     msg->is_pull = is_pull;
     msg->pull_size = pull_size;
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587

     /*Fill in simplenet information*/     
     if(is_last_pckt) /* Its the last packet so pass in remote event information*/
      {
       if(remote_event_size)
	 {
           msg->event_size_bytes = remote_event_size;
           memcpy(tmp_ptr, remote_event, remote_event_size);
           tmp_ptr += remote_event_size;
	 }
       if(self_event_size)
       {
	   msg->local_event_size_bytes = self_event_size;
	   memcpy(tmp_ptr, self_event, self_event_size);
	   tmp_ptr += self_event_size;
       }
      }
     tw_event_send(e_new);
588
     return xfer_to_nic_time;
589
590
}

591
static void sn_configure()
592
{
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
    anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
    assert(anno_map);
    num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
    all_params = malloc(num_params * sizeof(*all_params));
    for (uint64_t i = 0; i < anno_map->num_annos; i++){
        const char * anno = anno_map->annotations[i];
        int rc;
        rc = configuration_get_value_double(&config, "PARAMS",
                "net_startup_ns", anno, &all_params[i].net_startup_ns);
        if (rc != 0){
            tw_error(TW_LOC,
                    "simplenet: unable to read PARAMS:net_startup_ns@%s",
                    anno);
        }
        rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps",
                anno, &all_params[i].net_bw_mbps);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps@%s",
                    anno);
        }
    }
    if (anno_map->has_unanno_lp > 0){
        int rc;
        rc = configuration_get_value_double(&config, "PARAMS",
                "net_startup_ns", NULL,
                &all_params[num_params-1].net_startup_ns);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_startup_ns");
        }
        rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps",
                NULL, &all_params[num_params-1].net_bw_mbps);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps");
        }
    }
628
629
630
631
632
633
634
635
}

static void simplenet_packet_event_rc(tw_lp *sender)
{
    codes_local_latency_reverse(sender);
    return;
}

636
637
638
639
static tw_lpid sn_find_local_device(
        const char * annotation, 
        int          ignore_annotations,
        tw_lp      * sender)
640
{
641
642
     char lp_group_name[MAX_NAME_LENGTH];
     int mapping_rep_id, mapping_offset, dummy;
643
644
     tw_lpid dest_id;

645
646
647
     // TODO: don't ignore annotations
     codes_mapping_get_lp_info(sender->gid, lp_group_name, &dummy, NULL,
             &dummy, NULL, &mapping_rep_id, &mapping_offset);
648
649
     codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, annotation,
             ignore_annotations, mapping_rep_id, mapping_offset, &dest_id);
650
651
652
653

    return(dest_id);
}

654
#if SIMPLENET_DEBUG
655
656
657
658
659
void print_msg(sn_message *m){
    printf(" sn:\n  type:%d, magic:%d, src:%lu, dest:%lu, esize:%d, lsize:%d\n",
            m->event_type, m->magic, m->src_gid, m->final_dest_gid,
            m->event_size_bytes, m->local_event_size_bytes);
}
660
#endif
661

662

663
664
665
666
667
668
669
670
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */