resource statistics gathering

codes/resource.h

@@ -32,16 +32,40 @@ int resource_get(uint64_t req, resource_token_t tok, resource *r);
  * Returns 0 on success, 2 on invalid token */
 int resource_free(uint64_t req, resource_token_t tok, resource *r);
 
-/* Reservation functions, same return value as get. 
- * These functions expect exactly one caller per LP group as 
- * defined by the codes configuration 
+/* Determine amount of resource units remaining
+ * Returns 0 on success, 2 on invalid token */
+int resource_get_avail(resource_token_t tok, uint64_t *avail, resource *r);
+
+/* Determine amount of used resource units.
+ * The number returned is based on the pool-specific maximums, for which
+ * reserve calls can change */
+int resource_get_used(resource_token_t tok, uint64_t *used, resource *r);
+
+/* Get and restore minimum stat (support for RC). So that the resource
+ * interface doesn't need to upgrade to a full LP, store stats in
+ * caller-provided arguments. */
+int resource_get_min_avail(resource_token_t tok, uint64_t *min_avail,
+        resource *r);
+int resource_restore_min_avail(resource_token_t tok, uint64_t min_avail,
+        resource *r);
+
+/* Reservation functions, same return value as get.
+ * These functions expect exactly one caller per LP group as
+ * defined by the codes configuration
  * TODO: "un-reserving" not yet supported */
 int resource_reserve(uint64_t req, resource_token_t *tok, resource *r);
 
 #define MAX_RESERVE 8
 struct resource_s {
-    /* index 0 is the general pool, 1... are the reserved pools */
+    // index 0 is the general pool, 1... are the reserved pools
+    // current available 
     uint64_t avail[MAX_RESERVE+1];
+    // maximums per pool (the max for the general pool is reduced on reserve!)
+    uint64_t max[MAX_RESERVE+1];
+    // statistics - minimum available at any time
+    uint64_t min_avail[MAX_RESERVE+1];
+    // global maximum
+    uint64_t max_all;
     unsigned int num_tokens;
 };
 

src/util/resource-lp.c

@@ -9,6 +9,7 @@
 #include "codes/codes_mapping.h"
 #include "codes/jenkins-hash.h"
 #include "codes/quicklist.h"
+#include "codes/lp-io.h"
 #include "ross.h"
 #include <assert.h>
 #include <stdio.h>
@@ -73,6 +74,9 @@ struct resource_msg_internal{
 
 struct resource_msg {
     struct resource_msg_internal i, i_rc;
+    // for RC (asides from the message itself): the previous minimum resource
+    // value
+    uint64_t min_avail_rc;
 };
 
 struct pending_op {
@@ -183,6 +187,8 @@ static void handle_resource_get(
         tw_lp * lp){
     int ret = 1;
     int send_ack = 1;
+    // save the previous minimum for RC
+    assert(!resource_get_min_avail(m->i.tok, &m->min_avail_rc, &ns->r));
     if (!qlist_empty(&ns->pending[m->i.tok]) || 
             (ret = resource_get(m->i.req, m->i.tok, &ns->r))){
         /* failed to receive data */
@@ -223,8 +229,8 @@ static void handle_resource_get_rc(
     }
 
     if (b->c2){
-        int ret = resource_free(m->i.req, m->i.tok, &ns->r);
-        assert(ret != 2);
+        assert(!resource_restore_min_avail(m->i.tok, m->min_avail_rc, &ns->r));
+        assert(!resource_free(m->i.req, m->i.tok, &ns->r));
     }
 }
 
@@ -266,6 +272,7 @@ static void handle_resource_deq(
 
     struct qlist_head *front = ns->pending[m->i.tok].next;
     pending_op *p = qlist_entry(front, pending_op, ql);
+    assert(!resource_get_min_avail(m->i.tok, &m->min_avail_rc, &ns->r));
     int ret = resource_get(p->m.req, p->m.tok, &ns->r);
     assert(ret != 2);
     if (!ret){
@@ -303,6 +310,7 @@ static void handle_resource_deq_rc(
         op->m = m->i_rc;
         qlist_add(&op->ql, &ns->pending[m->i.tok]);
         resource_response_rc(lp);
+        assert(!resource_restore_min_avail(m->i.tok, m->min_avail_rc, &ns->r)); 
         assert(!resource_free(op->m.req, op->m.tok, &ns->r));
         /* reverse "deq next" op */
         codes_local_latency_reverse(lp);
@@ -377,9 +385,6 @@ void resource_rev_handler(
         default:
             assert(0);
     }
-
-    // NOTE: ross doesn't reset b in the case of multiple rollbacks...
-    *(int*)b = 0;
 }
 
 void resource_finalize(
@@ -387,11 +392,29 @@ void resource_finalize(
         tw_lp * lp){
     /* Fill me in... */
     struct qlist_head *ent;
-    qlist_for_each(ent, ns->pending){
-        pending_op *op = qlist_entry(ent, pending_op, ql);
-        fprintf(stderr, "WARNING: resource LP %lu has a pending allocation\n",
-                lp->gid);
+    for (int i = 0; i < MAX_RESERVE+1; i++){
+        qlist_for_each(ent, &ns->pending[i]){
+            fprintf(stderr, "WARNING: resource LP %lu has a pending allocation\n",
+                    lp->gid);
+        }
+    }
+
+    char *out_buf = malloc(1<<12);
+    int written;
+    // see if I'm the "first" resource
+    if (codes_mapping_get_lp_global_rel_id(lp->gid) == 0){
+        written = sprintf(out_buf, 
+                "# format: <LP> <max used general> <max used token...>\n");
+        lp_io_write(lp->gid, "resource", written, out_buf);
+    }
+    written = sprintf(out_buf, "%lu", lp->gid);
+
+    // TODO: wrap this up in the resource interface
+    for (int i = 0; i < ns->r.num_tokens+1; i++){
+        written += sprintf(out_buf+written, " %lu", ns->r.max[i]-ns->r.min_avail[i]);
     }
+    written += sprintf(out_buf+written, "\n");
+    lp_io_write(lp->gid, "resource", written, out_buf);
 }
 
 /**** END IMPLEMENTATIONS ****/

src/util/resource.c

@@ -6,11 +6,24 @@
 
 #include "codes/resource.h"
 #include <assert.h>
+#include <string.h>
+
+static uint64_t min_u64(uint64_t a, uint64_t b){
+    return a < b ? a : b;
+}
 
 /* initialize with avail capacity, all unreserved */
 void resource_init(uint64_t avail, resource *r){
-    r->avail[0] = avail;
     r->num_tokens = 0;
+    r->max_all  = avail;
+    r->avail[0] = avail;
+    r->max[0] = avail;
+    r->min_avail[0] = UINT64_MAX;
+    for (int i = 1; i < MAX_RESERVE; i++){
+        r->avail[i] = 0;
+        r->max[i] = 0;
+        r->min_avail[i] = UINT64_MAX;
+    }
 }
 
 /* Acquire req units of the resource. 
@@ -25,6 +38,7 @@ int resource_get(uint64_t req, resource_token_t tok, resource *r){
     }
     else{
         r->avail[tok] -= req;
+        r->min_avail[tok] = min_u64(r->avail[tok], r->min_avail[tok]);
         return 0;
     }
 }
@@ -37,22 +51,76 @@ int resource_free(uint64_t req, resource_token_t tok, resource *r){
     }
     else{
         r->avail[tok] += req;
+        // TODO: check for invalid state? (more available than possible)
         return 0;
     }
 }
 
-/* Reservation functions, same return value as get. 
+/* Determine amount of resource units remaining
+ * Returns 0 on success, 2 on invalid token */
+int resource_get_avail(resource_token_t tok, uint64_t *avail, resource *r){
+    if (tok > r->num_tokens){
+        return 2;
+    }
+    else{
+        return r->avail[tok];
+    }
+}
+
+/* Determine amount of used resource units.
+ * The number returned is based on the pool-specific maximums, for which
+ * reserve calls can change */
+int resource_get_used(resource_token_t tok, uint64_t *used, resource *r){
+    if (tok > r->num_tokens){
+        return 2;
+    }
+    else{
+        return r->max[tok] - r->avail[tok];
+    }
+}
+
+/* Get and restore minimum stat (support for RC). So that the resource
+ * interface doesn't need to upgrade to a full LP, store stats in
+ * caller-provided arguments. */
+int resource_get_min_avail(resource_token_t tok, uint64_t *min_avail,
+        resource *r){
+    if (tok > r->num_tokens){
+        return 2;
+    }
+    else{
+        *min_avail = r->min_avail[tok];
+        return 0;
+    }
+}
+int resource_restore_min_avail(resource_token_t tok, uint64_t min_avail,
+        resource *r){
+    if (tok > r->num_tokens){
+        return 2;
+    }
+    else{
+        r->min_avail[tok] = min_avail;
+        return 0;
+    }
+}
+
+/* Reservation function, same return values as get. 
  * These functions expect exactly one caller per LP group as 
  * defined by the codes configuration 
  * TODO: "un-reserving" not yet supported */
 int resource_reserve(uint64_t req, resource_token_t *tok, resource *r){
-    if (req > r->avail[0]){
+    if (r->num_tokens > MAX_RESERVE){
+        return 2;
+    }
+    else if (req > r->avail[0]){
         return 1;
     }
     else{
         /* reserved tokens start from 1 */ 
         *tok = ++(r->num_tokens);
         r->avail[*tok] = req;
+        r->max[*tok] = req;
+        r->max[0] -= req;
+        r->avail[0] -= req;
         return 0;
     }
 }