simplenet-upd.c 18.7 KB
Newer Older
1
/*
Philip Carns's avatar
Philip Carns committed
2
 * Copyright (C) 2013 University of Chicago.
3
 * See COPYRIGHT notice in top-level directory.
Philip Carns's avatar
Philip Carns committed
4
 *
5
6
7
8
9
10
 */

#include <string.h>
#include <assert.h>
#include <ross.h>

11
12
#include "codes/lp-io.h"
#include "codes/jenkins-hash.h"
13
#include "codes/model-net-method.h"
14
#include "codes/model-net.h"
15
#include "codes/model-net-lp.h"
16
17
#include "codes/codes_mapping.h"
#include "codes/codes.h"
18
#include "codes/net/simplenet-upd.h"
19
20
21
22

#define CATEGORY_NAME_MAX 16
#define CATEGORY_MAX 12

23
24
25
#define LP_CONFIG_NM (model_net_lp_config_names[SIMPLENET])
#define LP_METHOD_NM (model_net_method_names[SIMPLENET])

26
27
28
29
30
31
32
33
/* structs for initializing a network/ specifying network parameters */
struct simplenet_param
{
  double net_startup_ns; /*simplenet startup cost*/
  double net_bw_mbps; /*Link bandwidth per byte*/
};
typedef struct simplenet_param simplenet_param;

34
35
36
37
38
39
40
41
/*Define simplenet data types and structs*/
typedef struct sn_state sn_state;

struct sn_state
{
    /* next idle times for network card, both inbound and outbound */
    tw_stime net_send_next_idle;
    tw_stime net_recv_next_idle;
42
43
    const char * anno;
    simplenet_param params;
44
    struct mn_stats sn_stats_array[CATEGORY_MAX];
45
46
};

47
48
49
50
51
/* annotation-specific parameters (unannotated entry occurs at the 
 * last index) */
static uint64_t                  num_params = 0;
static simplenet_param         * all_params = NULL;
static const config_anno_map_t * anno_map   = NULL;
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

static int sn_magic = 0;

/* returns a pointer to the lptype struct to use for simplenet LPs */
static const tw_lptype* sn_get_lp_type(void);

/* retrieve the size of the portion of the event struct that is consumed by
 * the simplenet module.  The caller should add this value to the size of
 * its own event structure to get the maximum total size of a message.
 */
static int sn_get_msg_sz(void);

/* Returns the simplenet magic number */
static int sn_get_magic();

67
#if SIMPLENET_DEBUG
68
static void print_msg(sn_message *m);
69
#endif
70

71
72
73
74
75
76
/* collective network calls */
static void simple_net_collective();

/* collective network calls-- rc */
static void simple_net_collective_rc();

77
78
79
80
81
82
83
84
85
86
87
88
89
90
/* allocate a new event that will pass through simplenet to arriave at its
 * destination:
 *
 * - category: category name to associate with this communication
 * - final_dest_gid: the LP that the message should be delivered to.
 * - event_size_bytes: size of event msg that will be delivered to
 * final_dest_gid.
 * - local_event_size_byte: size of event message that will delivered to
 *   local LP upon local send comletion (set to 0 if not used)
 * - net_msg_size_bytes: size of simulated network message in bytes.
 * - sender: LP calling this function.
 */
/* Modelnet interface events */
/* sets up the simplenet parameters through modelnet interface */
91
static void sn_configure();
92
93

/* Issues a simplenet packet event call */
94
static tw_stime simplenet_packet_event(
95
96
     char* category, 
     tw_lpid final_dest_lp, 
97
     uint64_t packet_size, 
98
99
     int is_pull,
     uint64_t pull_size, /* only used when is_pull==1 */
100
     tw_stime offset,
101
102
103
104
     int remote_event_size, 
     const void* remote_event, 
     int self_event_size,
     const void* self_event,
105
     tw_lpid src_lp,
106
107
108
109
110
     tw_lp *sender,
     int is_last_pckt);
static void simplenet_packet_event_rc(tw_lp *sender);

static void simplenet_packet_event_rc(tw_lp *sender);
111
112
113

static void sn_report_stats();

114
115
116
117
static tw_lpid sn_find_local_device(
        const char * annotation, 
        int          ignore_annotations,
        tw_lp      * sender);
118

119
120
121
/* data structure for model-net statistics */
struct model_net_method simplenet_method =
{
122
    .mn_configure = sn_configure,
123
124
125
126
    .model_net_method_packet_event = simplenet_packet_event,
    .model_net_method_packet_event_rc = simplenet_packet_event_rc,
    .mn_get_lp_type = sn_get_lp_type,
    .mn_get_msg_sz = sn_get_msg_sz,
127
    .mn_report_stats = sn_report_stats,
128
    .model_net_method_find_local_device = sn_find_local_device,
129
130
    .mn_collective_call = simple_net_collective,	
    .mn_collective_call_rc = simple_net_collective_rc
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
};

static void sn_init(
    sn_state * ns,
    tw_lp * lp);
static void sn_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void sn_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void sn_finalize(
    sn_state * ns,
    tw_lp * lp);

tw_lptype sn_lp = {
     (init_f) sn_init,
     (event_f) sn_event,
     (revent_f) sn_rev_event,
     (final_f) sn_finalize,
     (map_f) codes_mapping,
     sizeof(sn_state),
};

159
static tw_stime rate_to_ns(uint64_t bytes, double MB_p_s);
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
static void handle_msg_ready_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_ready_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_start_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);
static void handle_msg_start_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp);

/* returns pointer to LP information for simplenet module */
static const tw_lptype* sn_get_lp_type()
{
    return(&sn_lp);
}

/* returns number of bytes that the simplenet module will consume in event
 * messages
 */
static int sn_get_msg_sz(void)
{
    return(sizeof(sn_message));
}
194
195
196
197
198
199
200
201
202
203
204
205
206
/* collective network calls */
static void simple_net_collective()
{
/* collectives not supported */
    return;
}

/* collective network call -- rc*/
static void simple_net_collective_rc()
{
/* collectives not supported */
   return;
}
207

208
209
210
211
212
213
/* report network statistics */
static void sn_report_stats()
{
   /* TODO: Do we have some simplenet statistics to report like we have for torus and dragonfly? */
   return;
}
214
215
216
217
static void sn_init(
    sn_state * ns,
    tw_lp * lp)
{
218
    uint32_t h1 = 0, h2 = 0;
219
220
221
222
223
224
    memset(ns, 0, sizeof(*ns));

    /* all devices are idle to begin with */
    ns->net_send_next_idle = tw_now(lp);
    ns->net_recv_next_idle = tw_now(lp);

225
226
227
228
229
230
231
232
    ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid);
    if (ns->anno == NULL)
        ns->params = all_params[num_params-1];
    else{
        int id = configuration_get_annotation_index(ns->anno, anno_map);
        ns->params = all_params[id];
    }

233
    bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
234
235
236
237
238
239
240
241
242
243
244
245
    sn_magic = h1+h2;
    /* printf("\n sn_magic %d ", sn_magic); */

    return;
}

static void sn_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
246
    assert(m->magic == sn_magic);
247
248
249

    switch (m->event_type)
    {
250
        case SN_MSG_START:
251
252
            handle_msg_start_event(ns, b, m, lp);
            break;
253
        case SN_MSG_READY:
254
255
256
257
258
259
260
261
262
263
264
265
266
267
            handle_msg_ready_event(ns, b, m, lp);
            break;
        default:
            assert(0);
            break;
    }
}

static void sn_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
268
    assert(m->magic == sn_magic);
269
270
271

    switch (m->event_type)
    {
272
        case SN_MSG_START:
273
274
            handle_msg_start_rev_event(ns, b, m, lp);
            break;
275
        case SN_MSG_READY:
276
277
278
279
280
281
282
283
284
285
286
287
288
289
            handle_msg_ready_rev_event(ns, b, m, lp);
            break;
        default:
            assert(0);
            break;
    }

    return;
}

static void sn_finalize(
    sn_state * ns,
    tw_lp * lp)
{
290
    model_net_print_stats(lp->gid, &ns->sn_stats_array[0]);
291
292
293
294
295
296
297
298
299
    return;
}

int sn_get_magic()
{
  return sn_magic;
}

/* convert MiB/s and bytes to ns */
300
static tw_stime rate_to_ns(uint64_t bytes, double MB_p_s)
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
{
    tw_stime time;

    /* bytes to MB */
    time = ((double)bytes)/(1024.0*1024.0);
    /* MB to s */
    time = time / MB_p_s;
    /* s to ns */
    time = time * 1000.0 * 1000.0 * 1000.0;

    return(time);
}

/* reverse computation for msg ready event */
static void handle_msg_ready_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
321
    struct mn_stats* stat;
322
323
324

    ns->net_recv_next_idle = m->net_recv_next_idle_saved;
    
325
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
326
327
    stat->recv_count--;
    stat->recv_bytes -= m->net_msg_size_bytes;
328
    stat->recv_time = m->recv_time_saved;
329

330
    if (m->event_size_bytes && m->is_pull){
331
        int net_id = model_net_get_id(LP_METHOD_NM);
332
333
334
        model_net_event_rc(net_id, lp, m->pull_size);
    }

335
336
337
338
339
340
341
342
343
344
345
346
347
    return;
}

/* handler for msg ready event.  This indicates that a message is available
 * to recv, but we haven't checked to see if the recv queue is available yet
 */
static void handle_msg_ready_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
    tw_stime recv_queue_time = 0;
348
    struct mn_stats* stat;
349
350
351

    //printf("handle_msg_ready_event(), lp %llu.\n", (unsigned long long)lp->gid);
    /* add statistics */
352
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
353
354
    stat->recv_count++;
    stat->recv_bytes += m->net_msg_size_bytes;
355
    m->recv_time_saved = stat->recv_time;
356
357
    stat->recv_time += rate_to_ns(m->net_msg_size_bytes,
            ns->params.net_bw_mbps);
358
359
360
361
362
363
364

    /* are we available to recv the msg? */
    /* were we available when the transmission was started? */
    if(ns->net_recv_next_idle > tw_now(lp))
        recv_queue_time += ns->net_recv_next_idle - tw_now(lp);

    /* calculate transfer time based on msg size and bandwidth */
365
366
    recv_queue_time += rate_to_ns(m->net_msg_size_bytes,
            ns->params.net_bw_mbps);
367
368
369
370
371
372
373
374

    /* bump up input queue idle time accordingly */
    m->net_recv_next_idle_saved = ns->net_recv_next_idle;
    ns->net_recv_next_idle = recv_queue_time + tw_now(lp);

    /* copy only the part of the message used by higher level */
    if(m->event_size_bytes)
    {
375
376
377
        //char *tmp_ptr = (char*)m;
        //tmp_ptr += sn_get_msg_sz();
        void *tmp_ptr = model_net_method_get_edata(SIMPLENET, m);
378
379
      /* schedule event to final destination for when the recv is complete */
//      printf("\n Remote message to LP %d ", m->final_dest_gid); 
380
381
        if (m->is_pull){
            /* call the model-net event */
382
            int net_id = model_net_get_id(LP_METHOD_NM);
383
384
385
386
387
            model_net_event(net_id, m->category, m->src_gid, m->pull_size,
                    recv_queue_time, m->event_size_bytes, tmp_ptr, 0, NULL,
                    lp);
        }
        else{
388
389
            tw_event * e_new = tw_event_new(m->final_dest_gid, recv_queue_time, lp);
            void * m_new = tw_event_data(e_new);
390
391
392
            memcpy(m_new, tmp_ptr, m->event_size_bytes);
            tw_event_send(e_new);
        }
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
    }

    return;
}

/* reverse computation for msg start event */
static void handle_msg_start_rev_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
    ns->net_send_next_idle = m->net_send_next_idle_saved;

    if(m->local_event_size_bytes > 0)
    {
        codes_local_latency_reverse(lp);
    }

412
413
    mn_stats* stat;
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
414
415
    stat->send_count--;
    stat->send_bytes -= m->net_msg_size_bytes;
416
    stat->send_time = m->send_time_saved;
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432

    return;
}

/* handler for msg start event; this indicates that the caller is trying to
 * transmit a message through this NIC
 */
static void handle_msg_start_event(
    sn_state * ns,
    tw_bf * b,
    sn_message * m,
    tw_lp * lp)
{
    tw_event *e_new;
    sn_message *m_new;
    tw_stime send_queue_time = 0;
433
    mn_stats* stat;
434
    int mapping_rep_id, mapping_offset, dummy;
435
    tw_lpid dest_id;
436
    char lp_group_name[MAX_NAME_LENGTH];
437
438
    int total_event_size;

439
440
    total_event_size = model_net_get_msg_sz(SIMPLENET) + m->event_size_bytes +
        m->local_event_size_bytes;
441
442
443

    //printf("handle_msg_start_event(), lp %llu.\n", (unsigned long long)lp->gid);
    /* add statistics */
444
    stat = model_net_find_stats(m->category, ns->sn_stats_array);
445
446
    stat->send_count++;
    stat->send_bytes += m->net_msg_size_bytes;
447
    m->send_time_saved = stat->send_time;
448
449
    stat->send_time += (ns->params.net_startup_ns + rate_to_ns(m->net_msg_size_bytes,
                ns->params.net_bw_mbps));
450
451
    if(stat->max_event_size < total_event_size)
        stat->max_event_size = total_event_size;
452
453

    /* calculate send time stamp */
454
    send_queue_time = ns->params.net_startup_ns; /* net msg startup cost */
455
456
457
458
459
460
461
462
463
    /* bump up time if the NIC send queue isn't idle right now */
    if(ns->net_send_next_idle > tw_now(lp))
        send_queue_time += ns->net_send_next_idle - tw_now(lp);

    /* move the next idle time ahead to after this transmission is
     * _complete_ from the sender's perspective 
     */ 
    m->net_send_next_idle_saved = ns->net_send_next_idle;
    ns->net_send_next_idle = send_queue_time + tw_now(lp) +
464
        rate_to_ns(m->net_msg_size_bytes, ns->params.net_bw_mbps);
465
466
467


    /* create new event to send msg to receiving NIC */
468
469
470
    // TODO: don't ignore annotations
    codes_mapping_get_lp_info(m->final_dest_gid, lp_group_name, &dummy, NULL,
            &dummy, NULL, &mapping_rep_id, &mapping_offset);
471
    codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, ns->anno, 0,
472
            mapping_rep_id, mapping_offset, &dest_id); 
473
474

//    printf("\n msg start sending to %d ", dest_id);
475
476
477
478
479
    void *m_data;
    //e_new = tw_event_new(dest_id, send_queue_time, lp);
    //m_new = tw_event_data(e_new);
    e_new = model_net_method_event_new(dest_id, send_queue_time, lp,
            SIMPLENET, (void**)&m_new, &m_data);
480
481
482
483

    /* copy entire previous message over, including payload from user of
     * this module
     */
484
485
486
487
488
489
490
491
    //memcpy(m_new, m, m->event_size_bytes + model_net_get_msg_sz(SIMPLENET));
    memcpy(m_new, m, sizeof(sn_message));
    if (m->event_size_bytes){
        memcpy(m_data, model_net_method_get_edata(SIMPLENET, m),
                m->event_size_bytes);
    }
    
    m_new->event_type = SN_MSG_READY;
492
    
493
494
    //print_base_from(SIMPLENET, m_new);
    //print_msg(m_new);
495
496
497
498
499
500
501
    tw_event_send(e_new);

    /* if there is a local event to handle, then create an event for it as
     * well
     */
    if(m->local_event_size_bytes > 0)
    {
502
        //char* local_event;
503

504
        e_new = tw_event_new(m->src_gid, send_queue_time+codes_local_latency(lp), lp);
505
506
        m_new = tw_event_data(e_new);

507
508
509
510
511
512
513
514
        void * m_loc = (char*) model_net_method_get_edata(SIMPLENET, m) +
            m->event_size_bytes;

         //local_event = (char*)m;
         //local_event += model_net_get_msg_sz(SIMPLENET) + m->event_size_bytes;         	 
        /* copy just the local event data over (which is past the remote event
         * in memory) */
        memcpy(m_new, m_loc, m->local_event_size_bytes);
515
516
517
518
519
520
521
522
523
        tw_event_send(e_new);
    }
    return;
}

/* Model-net function calls */

/*This method will serve as an intermediate layer between simplenet and modelnet. 
 * It takes the packets from modelnet layer and calls underlying simplenet methods*/
524
static tw_stime simplenet_packet_event(
525
526
		char* category,
		tw_lpid final_dest_lp,
527
		uint64_t packet_size,
528
529
                int is_pull,
                uint64_t pull_size, /* only used when is_pull == 1 */
530
                tw_stime offset,
531
532
533
534
		int remote_event_size,
		const void* remote_event,
		int self_event_size,
		const void* self_event,
535
                tw_lpid src_lp,
536
537
538
539
540
541
542
543
544
		tw_lp *sender,
		int is_last_pckt)
{
     tw_event * e_new;
     tw_stime xfer_to_nic_time;
     sn_message * msg;
     char* tmp_ptr;

     xfer_to_nic_time = codes_local_latency(sender);
545
546
547
     // this is a self message
     e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
             sender, SIMPLENET, (void**)&msg, (void**)&tmp_ptr);
548
549
     strcpy(msg->category, category);
     msg->final_dest_gid = final_dest_lp;
550
     msg->src_gid = src_lp;
551
552
553
554
     msg->magic = sn_get_magic();
     msg->net_msg_size_bytes = packet_size;
     msg->event_size_bytes = 0;
     msg->local_event_size_bytes = 0;
555
     msg->event_type = SN_MSG_START;
556
557
     msg->is_pull = is_pull;
     msg->pull_size = pull_size;
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575

     /*Fill in simplenet information*/     
     if(is_last_pckt) /* Its the last packet so pass in remote event information*/
      {
       if(remote_event_size)
	 {
           msg->event_size_bytes = remote_event_size;
           memcpy(tmp_ptr, remote_event, remote_event_size);
           tmp_ptr += remote_event_size;
	 }
       if(self_event_size)
       {
	   msg->local_event_size_bytes = self_event_size;
	   memcpy(tmp_ptr, self_event, self_event_size);
	   tmp_ptr += self_event_size;
       }
      }
     tw_event_send(e_new);
576
     return xfer_to_nic_time;
577
578
}

579
static void sn_configure()
580
{
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
    anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
    assert(anno_map);
    num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
    all_params = malloc(num_params * sizeof(*all_params));
    for (uint64_t i = 0; i < anno_map->num_annos; i++){
        const char * anno = anno_map->annotations[i];
        int rc;
        rc = configuration_get_value_double(&config, "PARAMS",
                "net_startup_ns", anno, &all_params[i].net_startup_ns);
        if (rc != 0){
            tw_error(TW_LOC,
                    "simplenet: unable to read PARAMS:net_startup_ns@%s",
                    anno);
        }
        rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps",
                anno, &all_params[i].net_bw_mbps);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps@%s",
                    anno);
        }
    }
    if (anno_map->has_unanno_lp > 0){
        int rc;
        rc = configuration_get_value_double(&config, "PARAMS",
                "net_startup_ns", NULL,
                &all_params[num_params-1].net_startup_ns);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_startup_ns");
        }
        rc = configuration_get_value_double(&config, "PARAMS", "net_bw_mbps",
                NULL, &all_params[num_params-1].net_bw_mbps);
        if (rc != 0){
            tw_error(TW_LOC, "simplenet: unable to read PARAMS:net_bw_mbps");
        }
    }
616
617
618
619
620
621
622
623
}

static void simplenet_packet_event_rc(tw_lp *sender)
{
    codes_local_latency_reverse(sender);
    return;
}

624
625
626
627
static tw_lpid sn_find_local_device(
        const char * annotation, 
        int          ignore_annotations,
        tw_lp      * sender)
628
{
629
630
     char lp_group_name[MAX_NAME_LENGTH];
     int mapping_rep_id, mapping_offset, dummy;
631
632
     tw_lpid dest_id;

633
634
635
     // TODO: don't ignore annotations
     codes_mapping_get_lp_info(sender->gid, lp_group_name, &dummy, NULL,
             &dummy, NULL, &mapping_rep_id, &mapping_offset);
636
637
     codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, annotation,
             ignore_annotations, mapping_rep_id, mapping_offset, &dest_id);
638
639
640
641

    return(dest_id);
}

642
#if SIMPLENET_DEBUG
643
644
645
646
647
void print_msg(sn_message *m){
    printf(" sn:\n  type:%d, magic:%d, src:%lu, dest:%lu, esize:%d, lsize:%d\n",
            m->event_type, m->magic, m->src_gid, m->final_dest_gid,
            m->event_size_bytes, m->local_event_size_bytes);
}
648
#endif
649

650

651
652
653
654
655
656
657
658
/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */