priority scheduler impl + modelnet msg param injection API

- N priorities processed in increasing order
- queue for priority i not touched until 0...i-1 queues are empty
- example injection API usage shown in test program
  - currently, only the priority scheduler has need of it

codes/model-net-lp.h

@@ -71,6 +71,9 @@ typedef struct model_net_base_msg {
         model_net_request req;
         struct {} sched; // needs nothing at the moment
     } u;
+    // parameters to pass to new messages (via model_net_set_msg_params)
+    // TODO: make this a union for multiple types of parameters
+    mn_sched_params sched_params;
     model_net_sched_rc rc; // rc for scheduling events
 } model_net_base_msg;
 

codes/model-net-sched.h

@@ -20,6 +20,7 @@
     X(MN_SCHED_FCFS,      "fcfs",        &fcfs_tab) \
     X(MN_SCHED_FCFS_FULL, "fcfs-full",   &fcfs_tab) \
     X(MN_SCHED_RR,        "round-robin", &rr_tab) \
+    X(MN_SCHED_PRIO,      "priority",    &prio_tab) \
     X(MAX_SCHEDS,         NULL,          NULL)
 
 #define X(a,b,c) a,
@@ -28,37 +29,84 @@ enum sched_type {
 };
 #undef X
 
-extern char * sched_names[];
-
 /// scheduler decls
 
 typedef struct model_net_sched_s model_net_sched;
 typedef struct model_net_sched_rc_s model_net_sched_rc;
 
+// priority scheduler configurtion parameters
+typedef struct mn_prio_params_s {
+    int num_prios; // number of priorities
+    // sub-scheduler to use. can be any but prio
+    enum sched_type sub_stype;
+} mn_prio_params;
+
+// TODO: other scheduler config params
+
+// initialization parameter set
+typedef struct model_net_sched_cfg_params_s {
+    enum sched_type type;
+    union {
+        mn_prio_params prio;
+    } u;
+} model_net_sched_cfg_params;
+
+typedef struct mn_sched_cfg_params {
+    mn_prio_params prio;
+} mn_sched_cfg_params;
+
+/// message-specific parameters
+enum sched_msg_param_type {
+    MN_SCHED_PARAM_PRIO, // currently, only the priority scheduler has params
+    MAX_SCHED_MSG_PARAM_TYPES
+};
+
+// scheduler-specific parameter definitions must go here
+typedef struct mn_sched_params_s {
+    int prio; // MN_SCHED_PARAM_PRIO (currently the only one)
+} mn_sched_params;
+
 /// interface to be implemented by schedulers
 /// see corresponding general functions
 typedef struct model_net_sched_interface {
-    void (*init)(struct model_net_method *method, void ** sched);
+    // initialize the scheduler
+    // params - scheduler specific params (currently only prio q uses)
+    void (*init)(
+            const struct model_net_method     * method, 
+            const model_net_sched_cfg_params  * params,
+            void                             ** sched);
+    // finalize the scheduler
     void (*destroy)(void * sched);
+    // add a new request to the scheduler
+    // sched_params - per-message parameters distinct to each scheduler:
+    //                prio (currently the only user): int priority
+    //              - NULL arguments should be treated as "use default value" 
     void (*add)(
-            model_net_request *req,
-            int remote_event_size,
-            void * remote_event,
-            int local_event_size,
-            void * local_event,
-            void *sched,
-            model_net_sched_rc *rc,
-            tw_lp *lp);
+            model_net_request  * req,
+            void               * sched_params,
+            int                  remote_event_size,
+            void               * remote_event,
+            int                  local_event_size,
+            void               * local_event,
+            void               * sched,
+            model_net_sched_rc * rc,
+            tw_lp              * lp);
+    // reverse the previous request addition
     void (*add_rc)(void *sched, model_net_sched_rc *rc, tw_lp *lp);
+    // schedule the next packet for processing by the model
     int  (*next)(
-            tw_stime *poffset,
-            void *sched,
+            tw_stime           * poffset,
+            void               * sched,
             // NOTE: copy here when deleting remote/local events for rc
-            void *rc_event_save,
-            model_net_sched_rc *rc,
-            tw_lp *lp);
-    void (*next_rc)(void * sched, void *rc_event_save, 
-            model_net_sched_rc *rc, tw_lp *lp);
+            void               * rc_event_save,
+            model_net_sched_rc * rc,
+            tw_lp              * lp);
+    // reverse schedule the previous packet
+    void (*next_rc)(
+            void               * sched,
+            void               * rc_event_save,
+            model_net_sched_rc * rc,
+            tw_lp              * lp);
 } model_net_sched_interface;
 
 /// overall scheduler struct - type puns the actual data structure
@@ -66,9 +114,11 @@ typedef struct model_net_sched_interface {
 struct model_net_sched_s {
     enum sched_type type;
     void * dat;
-    model_net_sched_interface *impl;
+    const model_net_sched_interface * impl;
 };
 
+/// scheduler-specific structures go here
+
 /// reverse computation structure - this is expected to be held by upper-level
 /// model-net LPs and passed to the scheduler functions to allow proper rc
 /// NOTE: since modelnet LPs will be stashing this in their event structs, 
@@ -78,10 +128,14 @@ struct model_net_sched_rc_s {
     // are equivalent 
     model_net_request req; // request gets deleted...
     int rtn; // return code from a sched_next 
+    int prio; // prio when doing priority queue events 
 };
 
+// initialize the scheduler
+// - params is created by the configuration routine and can be different from
+//   type to type. Currently only priority scheduler uses it
 void model_net_sched_init(
-        enum sched_type type, 
+        const model_net_sched_cfg_params * params,
         struct model_net_method *method,
         model_net_sched *sched);
 
@@ -108,8 +162,11 @@ void model_net_sched_next_rc(
 
 /// enter a new request into the scheduler, storing any info needed for rc in
 /// sched_rc
+/// sched_msg_params is scheduler-specific parameters (currently only used by
+/// prio scheduler)
 void model_net_sched_add(
         model_net_request *req,
+        void * sched_msg_params,
         int remote_event_size,
         void * remote_event,
         int local_event_size,
@@ -123,6 +180,8 @@ void model_net_sched_add_rc(
         model_net_sched_rc *sched_rc,
         tw_lp *lp);
 
+extern char * sched_names[];
+
 #endif /* end of include guard: MODEL_NET_SCHED_H */
 
 /*

codes/model-net.h

@@ -57,6 +57,13 @@ enum NETWORKS
 };
 #undef X
 
+// message parameter types
+enum msg_param_type {
+    // currently, scheduler parameters are the only type
+    MN_MSG_PARAM_SCHED,
+    MAX_MN_MSG_PARAM_TYPES
+};
+
 // network identifiers (both the config lp names and the model-net internal
 // names)
 extern char * model_net_lp_config_names[];
@@ -256,6 +263,28 @@ void model_net_pull_event_rc(
         int net_id,
         tw_lp *sender);
 
+/*
+ * Set message-specific parameters
+ * type     - overall type (see msg_param_type)
+ * sub_type - type of parameter specific to type. This is intended to be
+ *            an enum for each of msg_param_type's values
+ * params   - the parameter payload
+ *
+ * This function works by setting up a temporary parameter context within the
+ * model-net implementation (currently implemented as a set of translation-unit
+ * globals). Upon a subsequent model_net_*event* call, the context is consumed
+ * and reset to an unused state.
+ * 
+ * NOTE: this call MUST be placed in the same calling context as the subsequent
+ * model_net_*event* call. Otherwise, the parameters are not guaranteed to work
+ * on the intended event, and may possibly be consumed by another, unrelated
+ * event.
+ */
+void model_net_set_msg_param(
+        enum msg_param_type type,
+        int sub_type,
+        const void * params);
+
 /* returns pointer to LP information for simplenet module */
 const tw_lptype* model_net_get_lp_type(int net_id);
 

src/models/networks/model-net/model-net-lp.c

@@ -23,8 +23,8 @@ int model_net_base_magic;
 // issues...
 static int msg_offsets[MAX_NETS];
 
-typedef struct model_net_base_params {
-    enum sched_type stype;
+typedef struct model_net_base_params_s {
+    model_net_sched_cfg_params sched_params;
     uint64_t packet_size;
 } model_net_base_params;
 
@@ -145,29 +145,65 @@ static void base_read_config(const char * anno, model_net_base_params *p){
         int i;
         for (i = 0; i < MAX_SCHEDS; i++){
             if (strcmp(sched_names[i], sched) == 0){
-                p->stype = i;
+                p->sched_params.type = i;
                 break;
             }
         }
         if (i == MAX_SCHEDS){
             tw_error(TW_LOC,"Unknown value for PARAMS:modelnet-scheduler : "
-                    "%s\n", sched); 
+                    "%s", sched); 
         }
     }
     else{
         // default: FCFS
-        p->stype = MN_SCHED_FCFS;
+        p->sched_params.type = MN_SCHED_FCFS;
     }
 
-    if (p->stype == MN_SCHED_FCFS_FULL){
+    if (p->sched_params.type == MN_SCHED_FCFS_FULL){
         // override packet size to something huge (leave a bit in the unlikely
         // case that an op using packet size causes overflow)
         packet_size = 1ull << 62;
     }
-    else if (!packet_size && p->stype != MN_SCHED_FCFS_FULL)
+    else if (!packet_size && p->sched_params.type != MN_SCHED_FCFS_FULL)
     {
         packet_size = 512;
-        fprintf(stderr, "Warning, no packet size specified, setting packet size to %llu\n", packet_size);
+        fprintf(stderr, "Warning, no packet size specified, setting packet "
+                "size to %llu\n", packet_size);
+    }
+
+    // get scheduler-specific parameters
+    if (p->sched_params.type == MN_SCHED_PRIO){
+        // prio scheduler uses default parameters 
+        int             * num_prios = &p->sched_params.u.prio.num_prios;
+        enum sched_type * sub_stype = &p->sched_params.u.prio.sub_stype;
+        // number of priorities to allocate
+        ret = configuration_get_value_int(&config, "PARAMS",
+                "prio-sched-num-prios", anno, num_prios);
+        if (ret != 0)
+            *num_prios = 10;
+
+        ret = configuration_get_value(&config, "PARAMS",
+                "prio-sched-sub-sched", anno, sched, MAX_NAME_LENGTH);
+        if (ret == 0)
+            *sub_stype = MN_SCHED_FCFS;
+        else{
+            int i;
+            for (i = 0; i < MAX_SCHEDS; i++){
+                if (strcmp(sched_names[i], sched) == 0){
+                    *sub_stype = i;
+                    break;
+                }
+            }
+            if (i == MAX_SCHEDS){
+                tw_error(TW_LOC, "Unknown value for "
+                        "PARAMS:prio-sched-sub-sched %s", sched);
+            }
+            else if (i == MN_SCHED_PRIO){
+                tw_error(TW_LOC, "priority scheduler cannot be used as a "
+                        "priority scheduler's sub sched "
+                        "(PARAMS:prio-sched-sub-sched)");
+            }
+        }
     }
 
     p->packet_size = packet_size;
@@ -265,7 +301,7 @@ void model_net_base_lp_init(
 
     // TODO: parameterize scheduler type
     ns->sched = malloc(sizeof(model_net_sched));
-    model_net_sched_init(ns->params->stype, method_array[ns->net_id],
+    model_net_sched_init(&ns->params->sched_params, method_array[ns->net_id],
             ns->sched);
 
     ns->sub_type = model_net_get_lp_type(ns->net_id);
@@ -357,8 +393,22 @@ void handle_new_msg(
         local = m_data;
     }
     
-    model_net_sched_add(r, r->remote_event_size, remote, r->self_event_size,
-            local, ns->sched, &m->msg.m_base.rc, lp);
+    // set message-specific params
+    void * params = NULL;
+    switch(ns->sched->type){
+        case MN_SCHED_FCFS:
+        case MN_SCHED_FCFS_FULL:
+        case MN_SCHED_RR:
+            // no parameters
+            break;
+        case MN_SCHED_PRIO:
+            params = (void*)&m->msg.m_base.sched_params.prio;
+            break;
+        default:
+            assert(0);
+    }
+    model_net_sched_add(r, params, r->remote_event_size, remote,
+            r->self_event_size, local, ns->sched, &m->msg.m_base.rc, lp);
     
     if (ns->in_sched_loop == 0){
         b->c0 = 1;

src/models/networks/model-net/model-net-sched-impl.c

@@ -13,14 +13,6 @@
 #include "codes/quicklist.h"
 
 /// scheduler-specific data structures 
-/// NOTE: for now, scheduler data structures are the same - this may change in
-/// later versions
-
-typedef struct mn_sched {
-    // method containing packet event to call
-    struct model_net_method *method;
-    struct qlist_head reqs; // of type mn_sched_qitem
-} mn_sched;
 
 typedef struct mn_sched_qitem {
     model_net_request req;
@@ -33,61 +25,130 @@ typedef struct mn_sched_qitem {
     struct qlist_head ql;
 } mn_sched_qitem;
 
-/// scheduler-specific function decls and function table
+// fcfs and round-robin each use a single queue
+typedef struct mn_sched_queue {
+    // method containing packet event to call
+    const struct model_net_method *method;
+    struct qlist_head reqs; // of type mn_sched_qitem
+} mn_sched_queue;
+
+// priority scheduler consists of a bunch of rr/fcfs queues
+typedef struct mn_sched_prio {
+    mn_prio_params params;
+    const model_net_sched_interface *sub_sched_iface;
+    mn_sched_queue ** sub_scheds; // one for each params.num_prios
+} mn_sched_prio;
+
+/// scheduler-specific function decls and tables
 
 /// FCFS
 // void used to avoid ptr-to-ptr conv warnings
-static void fcfs_init (struct model_net_method *method, void ** sched);
+static void fcfs_init (
+        const struct model_net_method     * method, 
+        const model_net_sched_cfg_params  * params,
+        void                             ** sched);
 static void fcfs_destroy (void *sched);
 static void fcfs_add (
-        model_net_request *req, 
-        int remote_event_size,
-        void * remote_event,
-        int local_event_size,
-        void * local_event,
-        void *sched, 
-        model_net_sched_rc *rc, 
-        tw_lp *lp);
+        model_net_request  * req,
+        void               * sched_msg_params,
+        int                  remote_event_size,
+        void               * remote_event,
+        int                  local_event_size,
+        void               * local_event,
+        void               * sched,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
 static void fcfs_add_rc(void *sched, model_net_sched_rc *rc, tw_lp *lp);
-static int  fcfs_next(tw_stime *poffset, void *sched, void *rc_event_save, 
-        model_net_sched_rc *rc, tw_lp *lp);
-static void fcfs_next_rc(void *sched, void *rc_event_save,
-        model_net_sched_rc *rc, tw_lp *lp);
+static int  fcfs_next(
+        tw_stime           * poffset,
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
+static void fcfs_next_rc(
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
 
-static void rr_init (struct model_net_method *method, void ** sched);
+// ROUND-ROBIN
+static void rr_init (
+        const struct model_net_method     * method, 
+        const model_net_sched_cfg_params  * params,
+        void                             ** sched);
 static void rr_destroy (void *sched);
 static void rr_add (
-        model_net_request *req,
-        int remote_event_size,
-        void * remote_event,
-        int local_event_size,
-        void * local_event,
-        void *sched,
-        model_net_sched_rc *rc,
-        tw_lp *lp);
+        model_net_request  * req,
+        void               * sched_msg_params,
+        int                  remote_event_size,
+        void               * remote_event,
+        int                  local_event_size,
+        void               * local_event,
+        void               * sched,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
 static void rr_add_rc(void *sched, model_net_sched_rc *rc, tw_lp *lp);
-static int  rr_next(tw_stime *poffset, void *sched, void *rc_event_save,
-        model_net_sched_rc *rc, tw_lp *lp);
-static void rr_next_rc (void *sched, void *rc_event_save,
-        model_net_sched_rc *rc, tw_lp *lp);
+static int  rr_next(
+        tw_stime           * poffset,
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
+static void rr_next_rc (
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
+static void prio_init (
+        const struct model_net_method     * method, 
+        const model_net_sched_cfg_params  * params,
+        void                             ** sched);
+static void prio_destroy (void *sched);
+static void prio_add (
+        model_net_request  * req,
+        void               * sched_msg_params,
+        int                  remote_event_size,
+        void               * remote_event,
+        int                  local_event_size,
+        void               * local_event,
+        void               * sched,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
+static void prio_add_rc(void *sched, model_net_sched_rc *rc, tw_lp *lp);
+static int  prio_next(
+        tw_stime           * poffset,
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
+static void prio_next_rc (
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp);
 
 /// function tables (names defined by X macro in model-net-sched.h)
-static model_net_sched_interface fcfs_tab = 
+static const model_net_sched_interface fcfs_tab = 
 { &fcfs_init, &fcfs_destroy, &fcfs_add, &fcfs_add_rc, &fcfs_next, &fcfs_next_rc};
-static model_net_sched_interface rr_tab = 
+static const model_net_sched_interface rr_tab = 
 { &rr_init, &rr_destroy, &rr_add, &rr_add_rc, &rr_next, &rr_next_rc};
+static const model_net_sched_interface prio_tab =
+{ &prio_init, &prio_destroy, &prio_add, &prio_add_rc, &prio_next, &prio_next_rc};
 
 #define X(a,b,c) c,
-model_net_sched_interface * sched_interfaces[] = {
+const model_net_sched_interface * sched_interfaces[] = {
     SCHEDULER_TYPES
 };
 #undef X
 
 /// FCFS implementation 
 
-void fcfs_init(struct model_net_method *method, void ** sched){
-    *sched = malloc(sizeof(mn_sched));
-    mn_sched *ss = *sched;
+void fcfs_init(
+        const struct model_net_method     * method, 
+        const model_net_sched_cfg_params  * params,
+        void                             ** sched){
+    *sched = malloc(sizeof(mn_sched_queue));
+    mn_sched_queue *ss = *sched;
     ss->method = method;
     INIT_QLIST_HEAD(&ss->reqs);
 }
@@ -97,14 +158,15 @@ void fcfs_destroy(void *sched){
 }
 
 void fcfs_add (
-        model_net_request *req, 
-        int remote_event_size,
-        void * remote_event,
-        int local_event_size,
-        void * local_event,
-        void *sched, 
-        model_net_sched_rc *rc, 
-        tw_lp *lp){
+        model_net_request  * req,
+        void               * sched_msg_params,
+        int                  remote_event_size,
+        void               * remote_event,
+        int                  local_event_size,
+        void               * local_event,
+        void               * sched,
+        model_net_sched_rc * rc,
+        tw_lp              * lp){
     // NOTE: in optimistic mode, we currently do not have a good way to
     // reliably free and re-initialize the q item and the local/remote events
     // when processing next/next_rc events. Hence, the memory leaks. Later on
@@ -123,12 +185,12 @@ void fcfs_add (
         memcpy(q->local_event, local_event, local_event_size);
     }
     else { q->local_event = NULL; }
-    mn_sched *s = sched;
+    mn_sched_queue *s = sched;
     qlist_add_tail(&q->ql, &s->reqs);
 }
 
 void fcfs_add_rc(void *sched, model_net_sched_rc *rc, tw_lp *lp){
-    mn_sched *s = sched;
+    mn_sched_queue *s = sched;
     struct qlist_head *ent = qlist_pop_back(&s->reqs);
     assert(ent != NULL);
     mn_sched_qitem *q = qlist_entry(ent, mn_sched_qitem, ql);
@@ -138,9 +200,13 @@ void fcfs_add_rc(void *sched, model_net_sched_rc *rc, tw_lp *lp){
     free(q);
 }
 
-int fcfs_next(tw_stime *poffset, void *sched, void *rc_event_save,
-        model_net_sched_rc *rc, tw_lp *lp){
-    mn_sched *s = sched;
+int fcfs_next(
+        tw_stime           * poffset,
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp){
+    mn_sched_queue *s = sched;
     struct qlist_head *ent = s->reqs.next;
     if (ent == &s->reqs){
         rc->rtn = -1;
@@ -189,9 +255,12 @@ int fcfs_next(tw_stime *poffset, void *sched, void *rc_event_save,
     return rc->rtn;
 }
 
-void fcfs_next_rc(void *sched, void *rc_event_save, model_net_sched_rc *rc,
-        tw_lp *lp){
-    mn_sched *s = sched;
+void fcfs_next_rc(
+        void               * sched,
+        void               * rc_event_save,
+        model_net_sched_rc * rc,
+        tw_lp              * lp){
+    mn_sched_queue *s = sched;
     if (rc->rtn == -1){
         // no op
     }
@@ -233,26 +302,29 @@ void fcfs_next_rc(void *sched, void *rc_event_save, model_net_sched_rc *rc,
     }
 }
 
-void rr_init (struct model_net_method *method, void ** sched){
-    *sched = malloc(sizeof(mn_sched));
-    mn_sched *ss = *sched;
-    ss->method = method;
-    INIT_QLIST_HEAD(&ss->reqs);
+void rr_init (
+        const struct model_net_method     * method, 
+        const model_net_sched_cfg_params  * params,
+        void                             ** sched){
+    // same underlying representation
+    fcfs_init(method, params, sched);
 }
 
 void rr_destroy (void *sched){
-    free(sched);
+    // same underlying representation
+    fcfs_destroy(sched);
 }
 
 void rr_add (
-        model_net_request *req,
-        int remote_event_size,
-        void * remote_event,
-        int local_event_size,
-        void * local_event,
-        void *sched,
-        model_net_sched_rc *rc,
-        tw_lp *lp){
+        model_net_request  * req,
+        void               * sched_msg_params,
+        int                  remote_event_size,