Debugging performance issues. Fixed kv-server to set vsize=0 on error.

src/kv-client.c

@@ -131,7 +131,7 @@ hg_return_t kv_put(kv_context_t *context,
     st2 = ABT_get_wtime();
     ret = margo_forward(context->put_handle, &pin);
     et2 = ABT_get_wtime();
-    printf("kv_put forward time: %f microseconds\n", (et2-st2)*1000000);
+    printf("kv_put forward time: %f microseconds, vsize = %lu\n", (et2-st2)*1000000, vsize);
     assert(ret == HG_SUCCESS);
 
     ret = margo_get_output(context->put_handle, &pout);
@@ -165,7 +165,7 @@ hg_return_t kv_put(kv_context_t *context,
     st2 = ABT_get_wtime();
     ret = margo_forward(context->bulk_put_handle, &bpin);
     et2 = ABT_get_wtime();
-    printf("kv_put bulk forward time: %f microseconds\n", (et2-st2)*1000000);
+    printf("kv_put bulk forward time: %f microseconds, vsize = %lu\n", (et2-st2)*1000000, vsize);
     assert(ret == HG_SUCCESS);
 
     ret = margo_get_output(context->bulk_put_handle, &bpout);
@@ -210,7 +210,7 @@ hg_return_t kv_get(kv_context_t *context,
     st2 = ABT_get_wtime();
     ret = margo_forward(context->get_handle, &gin);
     et2 = ABT_get_wtime();
-    printf("kv_get forward time: %f microseconds\n", (et2-st2)*1000000);
+    printf("kv_get forward time: %f microseconds, vsize = %lu\n", (et2-st2)*1000000, size);
     assert(ret == HG_SUCCESS);
 
     ret = margo_get_output(context->get_handle, &gout);
@@ -247,7 +247,7 @@ hg_return_t kv_get(kv_context_t *context,
     st2 = ABT_get_wtime();
     ret = margo_forward(context->bulk_get_handle, &bgin);
     et2 = ABT_get_wtime();
-    printf("kv_get bulk forward time: %f microseconds\n", (et2-st2)*1000000);
+    printf("kv_get bulk forward time: %f microseconds, vsize = %lu\n", (et2-st2)*1000000, size);
     assert(ret == HG_SUCCESS);
 
     ret = margo_get_output(context->bulk_get_handle, &bgout);

src/kv-server.cc

@@ -229,6 +229,7 @@ static hg_return_t get_handler(hg_handle_t handle)
       gout.go.ret = HG_SUCCESS;
     }
     else {
+      gout.go.vsize = 0; // not returning a value
       gout.go.ret = HG_SIZE_ERROR; // caller should be checking return value
     }
   }
@@ -239,7 +240,7 @@ static hg_return_t get_handler(hg_handle_t handle)
 #ifdef KV_DEBUG
     std::cout << "get_handler datastore did not return a value" << std::endl;
 #endif
-    gout.go.vsize = 0;
+    gout.go.vsize = 0; // not returning a value
     gout.go.ret = HG_OTHER_ERROR; // caller should be checking return value
   }
 
@@ -305,13 +306,14 @@ static hg_return_t bulk_get_handler(hg_handle_t handle)
       bgout.ret = HG_SUCCESS;
     }
     else {
-      bgout.ret = HG_SIZE_ERROR;
+      bgout.size = 0; // not returning a value
+      bgout.ret = HG_SIZE_ERROR; // assuming caller will check return code
     }
   }
   else {
     // get on key did not find a value (return 0 for size)
-    bgout.size = 0; // assuming caller will check return code
-    bgout.ret = HG_OTHER_ERROR;
+    bgout.size = 0; // not returning a value
+    bgout.ret = HG_OTHER_ERROR; // assuming caller will check return code
   }
 	
 #ifdef KV_DEBUG

src/sds-keyval.h

@@ -42,7 +42,7 @@ typedef struct kv_context_s {
 } kv_context_t;
 
 
-#define MAX_RPC_MESSAGE_SIZE 4096 // in bytes
+#define MAX_RPC_MESSAGE_SIZE 4000 // in bytes
 
 // setup to support opaque type handling
 typedef char* kv_data_t;

test/test-mpi.cc

@@ -99,45 +99,46 @@ int main(int argc, char *argv[])
       hret = kv_open(context, server_addr_str, (char*)db);
       DIE_IF(hret != HG_SUCCESS, "kv_open");
       
-      // these puts/gets do explicit marshalling/unmarshalling as an example
-      // it's not necessary when dealing with simple types
+      size_t vsize = 1;
+      if (argc == 3) {
+	vsize = atoi(argv[2]);
+      }
+      printf("client (rank %d): using vsize = %lu, dsize = %lu\n", rank, vsize, vsize*sizeof(int32_t));
 
       // put
-      for (int i=1; i<rank*10; i++) {
-	int32_t key = 10*rank + i;
-	int32_t put_val = key;
-	std::vector<char> put_data;
-	put_data.resize(sizeof(put_val));
-	hg_size_t data_size = put_data.size();
-	memcpy(put_data.data(), &put_val, data_size);
+      for (int i=1; i<1000; i++) {
+	int32_t key = 1000*rank + i;
+	std::vector<int32_t> put_data;
+	put_data.resize(vsize, key);
+	hg_size_t data_size = put_data.size()*sizeof(int32_t); // size in char (bytes)
 
 	hret = kv_put(context, (void*)&key, sizeof(key),
 		      (void*)put_data.data(), data_size);
-	printf("(rank %d: put) key %d, value %d, size=%lu\n", rank, key, put_val, data_size);
+	printf("(rank %d: put) key %d, size=%lu\n", rank, key, data_size);
 	DIE_IF(hret != HG_SUCCESS, "kv_put");
       }
 
       sleep(2);
 
       // get
-      for (int i=1; i<rank*10; i++) {
-	int get_val;
-	int32_t key = 10*rank + i;
-	int32_t expected_get_val = key;
-	std::vector<char> get_data;
-	get_data.resize(sizeof(get_val));
-	hg_size_t data_size = get_data.size();
+      for (int i=1; i<1000; i++) {
+	int32_t key = 1000*rank + i;
+	std::vector<int32_t> expected_get_data;
+	expected_get_data.resize(vsize, key);
+	std::vector<int32_t> get_data;
+	get_data.resize(vsize);
+	hg_size_t data_size = get_data.size()*sizeof(int32_t); // size in char (bytes)
+	printf("(rank %d: get) key %d, size=%lu\n", rank, key, data_size);
 	hret = kv_get(context, (void*)&key, sizeof(key),
 		      (void*)get_data.data(), &data_size);
 	DIE_IF(hret != HG_SUCCESS, "kv_get");
 
-	get_data.resize(data_size);
-	memcpy(&get_val, get_data.data(), data_size);
-	if (expected_get_val == get_val) {
-	  printf("(rank %d: put/get succeeded) key %d, value %d, actual size=%lu\n", rank, key, get_val, data_size);
+	get_data.resize(data_size/sizeof(int32_t)); // size in int32_t
+	if (expected_get_data == get_data) {
+	  printf("(rank %d: put/get succeeded) key %d, actual size=%lu\n", rank, key, data_size);
 	}
 	else {
-	  printf("(rank %d: put/get failed) key %d, value %d, actual size=%lu\n", rank, key, get_val, data_size);
+	  printf("(rank %d: put/get failed) key %d, actual size=%lu\n", rank, key, data_size);
 	}
       }