config overhaul + annotation-aware mapping

All configuration now proceeds on a per-LP level, and requires separate
registration and configuration calls, as seen in the test programs.
model_net_set_params is no longer used, and is replaced by model_net_register
and model_net_configure. The dragonfly network, having two LP types bundled in
the same code-path, is special-cased in the registration code.

LP-mapping in model-net now has the following defaults:
- counts via codes_mapping_get_lp_count are now with respect to the calling
  network LP's annotation.
- when looking up network LPs via
  codes_mapping_get_lp_info/codes_mapping_get_lp_id, the annotation of the
  calling network LP is used. Hence, routing now occurs only between LPs of the
  same annotation. If the destination LP's group specified by model_net_*event
  does not contain a modelnet LP with the same annotation as the modelnet LP in
  the sender's group, then an error will occur (in codes_mapping).

Known Issues:
- modelnet users currently cannot specify which modelnet LP to use in the case
  of multiple modelnet LPs in the sender's group. This will be fixed in future
  commits after a consensus is reached on the best way to expose this
  information.

codes/model-net-lp.h

@@ -24,6 +24,12 @@
 
 extern int model_net_base_magic;
 
+// register the networks with ROSS, given the array of flags, one for each
+// network type
+void model_net_base_register(int *do_config_nets);
+// configure the base LP type, setting up general parameters
+void model_net_base_configure();
+
 // Construct a model-net-specific event, analagous to a tw_event_new and
 // codes_event_new. The difference here is that we return pointers to 
 // both the message data (to be cast into the appropriate type) and the
@@ -49,7 +55,7 @@ void * model_net_method_get_edata(int net_id, void * msg);
 /// use them
 
 // init method
-void model_net_base_init(int id_count, int *ids);
+void model_net_base_init(int *do_config_nets);
 
 enum model_net_base_event_type {
     MN_BASE_NEW_MSG,

codes/model-net-method.h

@@ -12,7 +12,7 @@
 struct model_net_method
 {
     uint64_t packet_size; /* packet size */
-    void (*mn_setup)(const void* net_params); /* For initializing the network */
+    void (*mn_configure)(); /* For initializing the network */
     tw_stime (*model_net_method_packet_event)(
         char* category, 
         tw_lpid final_dest_lp, 

codes/model-net-sched.h

@@ -30,10 +30,6 @@ enum sched_type {
 
 extern char * sched_names[];
 
-/// global for scheduler
-/// TODO: move away from using the global for when we have multiple networks
-extern enum sched_type mn_sched_type;
-
 /// scheduler decls
 
 typedef struct model_net_sched_s model_net_sched;

codes/model-net.h

@@ -24,7 +24,8 @@
  * messages as part of an event and expects FCFS ordering. A proper fix which
  * involves model-net LP-level scheduling of requests is ideal, but not 
  * feasible for now (would basically have to redesign model-net), so expose
- * explicit start-sequence and stop-sequence markers as a workaround */
+ * explicit start-sequence and stop-sequence markers as a workaround
+ */
 extern int in_sequence;
 extern tw_stime mn_msg_offset;
 #define MN_START_SEQ() do {\
@@ -36,11 +37,6 @@ extern tw_stime mn_msg_offset;
 } while (0)
 
 
-typedef struct simplenet_param simplenet_param;
-typedef struct simplewan_param simplewan_param;
-typedef struct dragonfly_param dragonfly_param;
-typedef struct torus_param torus_param;
-typedef struct loggp_param loggp_param;
 typedef struct mn_stats mn_stats;
 
 // use the X-macro to get types and names rolled up into one structure
@@ -93,53 +89,18 @@ struct mn_stats
     long max_event_size;
 };
 
-/* structs for initializing a network/ specifying network parameters */
-struct loggp_param
-{
-  char* net_config_file; /* file with loggp parameter table */
-};
-
-/* structs for initializing a network/ specifying network parameters */
-struct simplenet_param
-{
-  double net_startup_ns; /*simplenet startup cost*/
-  double net_bw_mbps; /*Link bandwidth per byte*/
-  int num_nics;
-};
-
-struct simplewan_param
-{
-    char bw_filename[MAX_NAME_LENGTH];
-    char startup_filename[MAX_NAME_LENGTH];
-};
+/* Registers all model-net LPs in ROSS. Should be called after 
+ * configuration_load, but before codes_mapping_setup */
+void model_net_register();
 
-struct dragonfly_param
-{
-  int num_routers; /*Number of routers in a group*/
-  double local_bandwidth;/* bandwidth of the router-router channels within a group */
-  double global_bandwidth;/* bandwidth of the inter-group router connections */
-  double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
-  int num_vcs; /* number of virtual channels */
-  int local_vc_size; /* buffer size of the router-router channels */
-  int global_vc_size; /* buffer size of the global channels */
-  int cn_vc_size; /* buffer size of the compute node channels */
-  short routing; /* minimal or non-minimal routing */
-  short chunk_size; /* full-sized packets are broken into smaller chunks.*/
-};
-
-struct torus_param
-{
-  int n_dims; /*Dimension of the torus network, 5-D, 7-D or any other*/
-  int* dim_length; /*Length of each torus dimension*/
-  double link_bandwidth;/* bandwidth for each torus link */
-  int buffer_size; /* number of buffer slots for each vc in flits*/
-  int num_vc; /* number of virtual channels for each torus link */
-  float mean_process;/* mean process time for each flit  */
-  int chunk_size; /* chunk is the smallest unit--default set to 32 */
-};
- /* In general we need to figure out how to pass configuration information to
- * the methods and we need to be able to calculate ross event message size.
- */
+/* Configures all model-net LPs based on the CODES configuration, and returns
+ * ids to address the different types by.
+ *
+ * id_count - the output number of networks
+ *
+ * return - the set of network IDs, indexed in the order given by the
+ * modelnet_order configuration parameter */
+int* model_net_configure(int *id_count);
 
 /* Initialize/configure the network(s) based on the CODES configuration.
  * returns an array of the network ids, indexed in the order given by the 

src/models/networks/model-net/loggp.c

@@ -26,15 +26,6 @@
 /*Define loggp data types and structs*/
 typedef struct loggp_state loggp_state;
 
-
-struct loggp_state
-{
-    /* next idle times for network card, both inbound and outbound */
-    tw_stime net_send_next_idle;
-    tw_stime net_recv_next_idle;
-    struct mn_stats loggp_stats_array[CATEGORY_MAX];
-};
-
 /* loggp parameters for a given msg size, as reported by netgauge */
 struct param_table_entry
 {
@@ -50,8 +41,32 @@ struct param_table_entry
     double G;
     double lsqu_gG;
 };
-struct param_table_entry param_table[100];
-static int param_table_size = 0;
+typedef struct param_table_entry param_table_entry;
+
+typedef struct loggp_param loggp_param;
+// loggp parameters
+struct loggp_param
+{
+    int table_size;
+    param_table_entry table[100];
+};
+
+
+struct loggp_state
+{
+    /* next idle times for network card, both inbound and outbound */
+    tw_stime net_send_next_idle;
+    tw_stime net_recv_next_idle;
+    const char * anno;
+    const loggp_param *params;
+    struct mn_stats loggp_stats_array[CATEGORY_MAX];
+};
+
+/* annotation-specific parameters (unannotated entry occurs at the 
+ * last index) */
+static uint64_t                  num_params = 0;
+static loggp_param             * all_params = NULL;
+static const config_anno_map_t * anno_map   = NULL;
 
 static int loggp_magic = 0;
 
@@ -73,21 +88,11 @@ static void loggp_collective();
 /* collective network calls-- rc */
 static void loggp_collective_rc();
 
-/* allocate a new event that will pass through loggp to arriave at its
- * destination:
- *
- * - category: category name to associate with this communication
- * - final_dest_gid: the LP that the message should be delivered to.
- * - event_size_bytes: size of event msg that will be delivered to
- * final_dest_gid.
- * - local_event_size_byte: size of event message that will delivered to
- *   local LP upon local send comletion (set to 0 if not used)
- * - net_msg_size_bytes: size of simulated network message in bytes.
- * - sender: LP calling this function.
- */
 /* Modelnet interface events */
 /* sets up the loggp parameters through modelnet interface */
-static void loggp_setup(const void* net_params);
+static void loggp_configure();
+
+static void loggp_set_params(const char * config_file, loggp_param * params);
 
 /* Issues a loggp packet event call */
 static tw_stime loggp_packet_event(
@@ -112,12 +117,14 @@ static void loggp_report_stats();
 
 static tw_lpid loggp_find_local_device(tw_lp *sender);
 
-static struct param_table_entry* find_params(uint64_t msg_size);
+static const struct param_table_entry* find_params(
+        uint64_t msg_size,
+        const loggp_param *params);
 
 /* data structure for model-net statistics */
 struct model_net_method loggp_method =
 {
-    .mn_setup = loggp_setup,
+    .mn_configure = loggp_configure,
     .model_net_method_packet_event = loggp_packet_event,
     .model_net_method_packet_event_rc = loggp_packet_event_rc,
     .mn_get_lp_type = loggp_get_lp_type,
@@ -220,6 +227,14 @@ static void loggp_init(
     ns->net_send_next_idle = tw_now(lp);
     ns->net_recv_next_idle = tw_now(lp);
 
+    ns->anno = codes_mapping_get_annotation_by_lpid(lp->gid);
+    if (ns->anno == NULL)
+        ns->params = &all_params[num_params-1];
+    else{
+        int id = configuration_get_annotation_index(ns->anno, anno_map);
+        ns->params = &all_params[id];
+    }
+
     bj_hashlittle2(LP_METHOD_NM, strlen(LP_METHOD_NM), &h1, &h2);
     loggp_magic = h1+h2;
     /* printf("\n loggp_magic %d ", loggp_magic); */
@@ -324,9 +339,9 @@ static void handle_msg_ready_event(
     loggp_message *m_new;
     struct mn_stats* stat;
     double recv_time;
-    struct param_table_entry *param;
+    const struct param_table_entry *param;
 
-    param = find_params(m->net_msg_size_bytes);
+    param = find_params(m->net_msg_size_bytes, ns->params);
 
     recv_time = ((double)(m->net_msg_size_bytes-1)*param->G);
     /* scale to nanoseconds */
@@ -420,9 +435,9 @@ static void handle_msg_start_event(
     char lp_group_name[MAX_NAME_LENGTH];
     int total_event_size;
     double xmit_time;
-    struct param_table_entry *param;
+    const struct param_table_entry *param;
 
-    param = find_params(m->net_msg_size_bytes);
+    param = find_params(m->net_msg_size_bytes, ns->params);
 
     total_event_size = model_net_get_msg_sz(LOGGP) + m->event_size_bytes +
         m->local_event_size_bytes;
@@ -465,7 +480,7 @@ static void handle_msg_start_event(
     // TODO: make annotation-aware
     codes_mapping_get_lp_info(m->final_dest_gid, lp_group_name, &mapping_grp_id,
             NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
-    codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, NULL, 1,
+    codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM, ns->anno, 0,
             mapping_rep_id, mapping_offset, &dest_id); 
 
     void *m_data;
@@ -574,52 +589,76 @@ static tw_stime loggp_packet_event(
      return xfer_to_nic_time;
 }
 
-static void loggp_setup(const void* net_params)
-{
-    loggp_param* loggp_params = (loggp_param*)net_params;
+static void loggp_configure(){
+    char config_file[MAX_NAME_LENGTH];
+
+    anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM);
+    assert(anno_map);
+    num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
+    all_params = malloc(num_params * sizeof(*all_params));
+
+    for (uint64_t i = 0; i < anno_map->num_annos; i++){
+        const char * anno = anno_map->annotations[i];
+        int rc = configuration_get_value_relpath(&config, "PARAMS",
+                "net_config_file", anno, config_file, MAX_NAME_LENGTH);
+        if (rc == 0){
+            tw_error(TW_LOC, "unable to read PARAMS:net_config_file@%s",
+                    anno);
+        }
+        loggp_set_params(config_file, &all_params[i]);
+    }
+    if (anno_map->has_unanno_lp > 0){
+        int rc = configuration_get_value_relpath(&config, "PARAMS",
+                "net_config_file", NULL, config_file, MAX_NAME_LENGTH);
+        if (rc == 0){
+            tw_error(TW_LOC, "unable to read PARAMS:net_config_file");
+        }
+        loggp_set_params(config_file, &all_params[anno_map->num_annos]);
+    }
+}
+
+void loggp_set_params(const char * config_file, loggp_param * params){
     FILE *conf;
     int ret;
     char buffer[512];
     int line_nr = 0;
+    printf("Loggp configured to use parameters from file %s\n", config_file);
 
-    /* TODO: update this to read on one proc and then broadcast */
-
-    printf("Loggp configured to use parameters from file %s\n", loggp_params->net_config_file);
-
-    conf = fopen(loggp_params->net_config_file, "r");
+    conf = fopen(config_file, "r");
     if(!conf)
     {
         perror("fopen");
         assert(0);
     }
 
+    params->table_size = 0;
     while(fgets(buffer, 512, conf))
     {
         line_nr++;
         if(buffer[0] == '#')
             continue;
         ret = sscanf(buffer, "%llu %d %lf %lf %lf %lf %lf %lf %lf %lf %lf", 
-            &param_table[param_table_size].size,
-            &param_table[param_table_size].n,
-            &param_table[param_table_size].PRTT_10s,
-            &param_table[param_table_size].PRTT_n0s,
-            &param_table[param_table_size].PRTT_nPRTT_10ss,
-            &param_table[param_table_size].L,
-            &param_table[param_table_size].o_s,
-            &param_table[param_table_size].o_r,
-            &param_table[param_table_size].g,
-            &param_table[param_table_size].G,
-            &param_table[param_table_size].lsqu_gG);
+            &params->table[params->table_size].size,
+            &params->table[params->table_size].n,
+            &params->table[params->table_size].PRTT_10s,
+            &params->table[params->table_size].PRTT_n0s,
+            &params->table[params->table_size].PRTT_nPRTT_10ss,
+            &params->table[params->table_size].L,
+            &params->table[params->table_size].o_s,
+            &params->table[params->table_size].o_r,
+            &params->table[params->table_size].g,
+            &params->table[params->table_size].G,
+            &params->table[params->table_size].lsqu_gG);
         if(ret != 11)
         {
             fprintf(stderr, "Error: malformed line %d in %s\n", line_nr, 
-                loggp_params->net_config_file);
+                config_file);
             assert(0);
         }
-        param_table_size++;
+        params->table_size++;
     }
 
-    printf("Parsed %d loggp table entries.\n", param_table_size);
+    printf("Parsed %d loggp table entries.\n", params->table_size);
 
     fclose(conf);
  
@@ -634,25 +673,26 @@ static void loggp_packet_event_rc(tw_lp *sender)
 
 /* find the parameters corresponding to the message size we are transmitting
  */
-static struct param_table_entry* find_params(uint64_t msg_size)
-{
+static const struct param_table_entry* find_params(
+        uint64_t msg_size,
+        const loggp_param *params) {
     int i;
 
     /* pick parameters based on the next smallest size in the table, but
      * default to beginning or end of table if we are out of range
      */
 
-    for(i=0; i<param_table_size; i++)
+    for(i=0; i < params->table_size; i++)
     {
-        if(param_table[i].size > msg_size)
+        if(params->table[i].size > msg_size)
         {
             break;
         }
     }
-    if(i>=param_table_size)
-        i = param_table_size-1;
+    if(i>=params->table_size)
+        i = params->table_size-1;
 
-    return(&param_table[i]);
+    return(&params->table[i]);
 }
 
 static tw_lpid loggp_find_local_device(tw_lp *sender)

src/models/networks/model-net/model-net-lp.c

@@ -23,18 +23,32 @@ int model_net_base_magic;
 // issues...
 static int msg_offsets[MAX_NETS];
 
+typedef struct model_net_base_params {
+    enum sched_type stype;
+    uint64_t packet_size;
+} model_net_base_params;
+
+/* annotation-specific parameters (unannotated entry occurs at the 
+ * last index) */
+static int                       num_params = 0;
+static const char              * annos[CONFIGURATION_MAX_ANNOS];
+static model_net_base_params     all_params[CONFIGURATION_MAX_ANNOS];
+
 typedef struct model_net_base_state {
     int net_id;
     // whether scheduler loop is running
     int in_sched_loop;
     // model-net scheduler
     model_net_sched *sched;
+    // parameters
+    const model_net_base_params * params;
     // lp type and state of underlying model net method - cache here so we
     // don't have to constantly look up
     const tw_lptype *sub_type;
     void *sub_state;
 } model_net_base_state;
 
+
 /**** END SIMULATION DATA STRUCTURES ****/
 
 /**** BEGIN LP, EVENT PROCESSING FUNCTION DECLS ****/
@@ -93,44 +107,132 @@ tw_lptype model_net_base_lp = {
 
 /**** BEGIN IMPLEMENTATIONS ****/
 
-void model_net_base_init(int id_count, int *ids){
+void model_net_base_register(int *do_config_nets){
+    // here, we initialize ALL lp types to use the base type
+    for (int i = 0; i < MAX_NETS; i++){
+        if (do_config_nets[i]){
+            lp_type_register(model_net_lp_config_names[i], &model_net_base_lp);
+            // HACK: unfortunately, we need to special-case dragonfly 
+            // registration at the moment - there are two LPs, and
+            // previously the LP matched on configuration initialized
+            // the "router" LP. Now that the base LP is in charge of
+            // registration, we need to take care of it
+            // TODO: fix the interface to have underlying networks do
+            // the registration
+            if (i==DRAGONFLY){
+                lp_type_register("dragonfly_router",
+                        &method_array[DRAGONFLY]->mn_get_lp_type()[1]);
+            }
+        }
+    }
+}
+
+static void base_read_config(const char * anno, model_net_base_params *p){
+    char sched[MAX_NAME_LENGTH];
+    long int packet_size_l = 0;
+    uint64_t packet_size;
+    int ret;
+
+    // TODO: make this annotation-specific - put in the config loop, make part
+    // of model-net base
+    ret = configuration_get_value(&config, "PARAMS", "modelnet_scheduler",
+            anno, sched, MAX_NAME_LENGTH);
+    configuration_get_value_longint(&config, "PARAMS", "packet_size", anno,
+            &packet_size_l);
+    packet_size = packet_size_l;
+
+    if (ret > 0){
+        int i;
+        for (i = 0; i < MAX_SCHEDS; i++){
+            if (strcmp(sched_names[i], sched) == 0){
+                p->stype = i;
+                break;
+            }
+        }
+        if (i == MAX_SCHEDS){
+            tw_error(TW_LOC,"Unknown value for PARAMS:modelnet-scheduler : "
+                    "%s\n", sched); 
+        }
+    }
+    else{
+        // default: FCFS
+        p->stype = MN_SCHED_FCFS;
+    }
+
+    if (p->stype == MN_SCHED_FCFS_FULL){
+        // override packet size to something huge (leave a bit in the unlikely
+        // case that an op using packet size causes overflow)
+        packet_size = 1ull << 62;
+    }
+    else if (!packet_size && p->stype != MN_SCHED_FCFS_FULL)
+    {
+        packet_size = 512;
+        fprintf(stderr, "Warning, no packet size specified, setting packet size to %llu\n", packet_size);
+    }
+
+    p->packet_size = packet_size;
+}
+
+void model_net_base_configure(){
     uint32_t h1=0, h2=0;
 
     bj_hashlittle2(MN_NAME, strlen(MN_NAME), &h1, &h2);
     model_net_base_magic = h1+h2;
 
-    // here, we initialize ALL lp types to use the base type
-    // TODO: only initialize ones that are actually used
-    for (int i = 0; i < id_count; i++){
-        lp_type_register(model_net_lp_config_names[ids[i]], &model_net_base_lp);
+    // set up offsets - doesn't matter if they are actually used or not
+    msg_offsets[SIMPLENET] =
+        offsetof(model_net_wrap_msg, msg.m_snet);
+    msg_offsets[SIMPLEWAN] =
+        offsetof(model_net_wrap_msg, msg.m_swan);
+    msg_offsets[TORUS] =
+        offsetof(model_net_wrap_msg, msg.m_torus);
+    msg_offsets[DRAGONFLY] =
+        offsetof(model_net_wrap_msg, msg.m_dfly);
+    msg_offsets[LOGGP] =
+        offsetof(model_net_wrap_msg, msg.m_loggp);
+
+    // perform the configuration(s)
+    // This part is tricky, as we basically have to look up all annotations that
+    // have LP names of the form modelnet_*. For each of those, we need to read
+    // the base parameters
+    // - the init is a little easier as we can use the LP-id to look up the
+    // annotation
+
+    // first grab all of the annotations and store locally
+    for (int c = 0; c < lpconf.lpannos_count; c++){
+        const config_anno_map_t *amap = &lpconf.lpannos[c];
+        if (strncmp("modelnet_", amap->lp_name, 9) == 0){
+            for (int n = 0; n < amap->num_annos; n++){
+                int a;
+                for (a = 0; a < num_params; a++){
+                    if (annos[a] != NULL &&
+                            strcmp(amap->annotations[n], annos[a]) == 0){
+                        break;
+                    }
+                }
+                if (a == num_params){
+                    // found a new annotation
+                    annos[num_params++] = amap->annotations[n];
+                }
+            }
+            if (amap->has_unanno_lp){
+                int a;
+                for (a = 0; a < num_params; a++){
+                    if (annos[a] == NULL)
+                        break;
+                }
+                if (a == num_params){
+                    // found a new (empty) annotation
+                    annos[num_params++] = NULL;
+                }
+            }
+        }
     }
 
-    // initialize the msg-specific offsets
-    for (int i = 0; i < id_count; i++){
-        switch(ids[i]){
-            case SIMPLENET:
-                msg_offsets[SIMPLENET] = 
-                    offsetof(model_net_wrap_msg, msg.m_snet);
-                break;
-            case SIMPLEWAN:
-                msg_offsets[SIMPLEWAN] = 
-                    offsetof(model_net_wrap_msg, msg.m_swan);
-                break;
-            case TORUS:
-                msg_offsets[TORUS] = 
-                    offsetof(model_net_wrap_msg, msg.m_torus);
-                break;
-            case DRAGONFLY:
-                msg_offsets[DRAGONFLY] = 
-                    offsetof(model_net_wrap_msg, msg.m_dfly);
-                break;
-            case LOGGP:
-                msg_offsets[LOGGP] = 
-                    offsetof(model_net_wrap_msg, msg.m_loggp);
-                break;
-            default:
-                assert(0);
-        }
+    // now that we have all of the annos for all of the networks, loop through
+    // and read the configs
+    for (int i = 0; i < num_params; i++){
+        base_read_config(annos[i], &all_params[i]);
     }
 }
 
@@ -138,14 +240,20 @@ void model_net_base_lp_init(
         model_net_base_state * ns,
         tw_lp * lp){
     // obtain the underlying lp type through codes-mapping
-    char lp_type_name[MAX_NAME_LENGTH];
+    char lp_type_name[MAX_NAME_LENGTH], anno[MAX_NAME_LENGTH];
     int dummy;
 
-    // TODO: ignoring annotation to get things up and running, but we'll want
-    // to actually have a sane design here at some point
-