From 20b130ca894985ed377fc89d2e6f02e91d675802 Mon Sep 17 00:00:00 2001 From: Matthieu Dorier Date: Thu, 17 Oct 2019 11:49:35 +0100 Subject: [PATCH] removed old tests --- old-test/bench-client.cc | 105 ---------------- old-test/test-cci.conf | 4 - old-test/test-client.cc | 112 ----------------- old-test/test-mpi-cci.conf | 4 - old-test/test-mpi-group.cc | 201 ------------------------------ old-test/test-mpi-srun-hosts.conf | 2 - old-test/test-mpi-srun-mp.conf | 2 - old-test/test-mpi.cc | 177 -------------------------- old-test/test-server.cc | 18 --- 9 files changed, 625 deletions(-) delete mode 100644 old-test/bench-client.cc delete mode 100644 old-test/test-cci.conf delete mode 100644 old-test/test-client.cc delete mode 100644 old-test/test-mpi-cci.conf delete mode 100644 old-test/test-mpi-group.cc delete mode 100644 old-test/test-mpi-srun-hosts.conf delete mode 100644 old-test/test-mpi-srun-mp.conf delete mode 100644 old-test/test-mpi.cc delete mode 100644 old-test/test-server.cc diff --git a/old-test/bench-client.cc b/old-test/bench-client.cc deleted file mode 100644 index 903be87..0000000 --- a/old-test/bench-client.cc +++ /dev/null @@ -1,105 +0,0 @@ -#include -#include - -#include -#include - -void RandomInsertSpeedTest(kv_database_t *db, - size_t key_num, bench_result_t *results) -{ - hg_return_t ret; - std::random_device r{}; - std::default_random_engine e1(r()); - std::uniform_int_distribution uniform_dist(0, key_num - 1); - - std::chrono::time_point start, end; - - // We loop for keynum * 2 because in average half of the insertion - // will hit an empty slot - start = std::chrono::system_clock::now(); - for(size_t i = 0;i < key_num * 2;i++) { - int32_t key = uniform_dist(e1); - ret = kv_put(db, &key, sizeof(key), &key, sizeof(key)); - assert(ret == HG_SUCCESS); - } - end = std::chrono::system_clock::now(); - - std::chrono::duration elapsed_seconds = end - start; - - results->nkeys = key_num *2; - results->insert_time = elapsed_seconds.count(); - - // Then test random read after random insert - int32_t value; - hg_size_t vsize; - start = std::chrono::system_clock::now(); - for(size_t i = 0;i < key_num * 2;i++) { - int32_t key = uniform_dist(e1); - vsize = sizeof(value); - ret = kv_get(db, &key, sizeof(key), &value, &vsize); - // key might not be in store due to randomness - // HG_OTHER_ERROR is basically a "key not found" return code - assert(ret == HG_SUCCESS || ret == HG_OTHER_ERROR); - } - end = std::chrono::system_clock::now(); - - elapsed_seconds = end - start; - results->read_time = elapsed_seconds.count(); - - results->overhead = 0; - - return; -} - -void print_results(bench_result_t *r) -{ - printf("inserts: %zd keys in %f seconds: %f Million-inserts per sec: %f usec per insert\n", - r->nkeys, r->insert_time, r->nkeys/(r->insert_time*1000*1000), - (r->insert_time*1000*1000)/r->nkeys); - printf("reads: %zd keys in %f seconds: %f Million-reads per sec: %f usec per read\n", - r->nkeys, r->read_time, r->nkeys/(r->read_time*1000*1000), - (r->read_time*1000*1000)/r->nkeys); -} - -int main(int argc, char **argv) -{ - hg_return_t ret; - bench_result_t rpc; - kv_context_t *context; - kv_database_t *db; - - assert(argc == 3); - size_t items = atoi(argv[1]); - char *server_addr_str = argv[2]; - - char *proto = kv_protocol(server_addr_str); - margo_instance_id mid = margo_init(proto, MARGO_CLIENT_MODE, 0, -1); - free(proto); - context = kv_client_register(mid); - - kv_db_type_t db_type = KVDB_BWTREE; // XXX make that an argument from argv - db = kv_open(context, server_addr_str, "db/testdb", db_type); - assert(db != NULL); - - RandomInsertSpeedTest(db, items, &rpc); - print_results(&rpc); - - bench_result_t *server; - server = kv_benchmark(db, items); - print_results(server); - free(server); - - /* close */ - ret = kv_close(db); - assert(ret == HG_SUCCESS); - - /* signal server */ - ret = kv_client_signal_shutdown(db); - assert(ret == HG_SUCCESS); - - /* cleanup */ - ret = kv_client_deregister(context); - assert(ret == HG_SUCCESS); - - margo_finalize(mid); -} diff --git a/old-test/test-cci.conf b/old-test/test-cci.conf deleted file mode 100644 index f569a01..0000000 --- a/old-test/test-cci.conf +++ /dev/null @@ -1,4 +0,0 @@ -[ethernet] -# use this example for TCP -transport = tcp -interface = eth4 # switch this to eth2 or to an external hostname for non-localhost use diff --git a/old-test/test-client.cc b/old-test/test-client.cc deleted file mode 100644 index 91e0f5e..0000000 --- a/old-test/test-client.cc +++ /dev/null @@ -1,112 +0,0 @@ -#include "sds-keyval.h" -#include -#include -#include -#include - -int main(int argc, char **argv) { - hg_return_t ret; - - assert(argc == 2); - char *server_addr_str = argv[1]; - - char *proto = kv_protocol(server_addr_str); - margo_instance_id mid = margo_init(proto, MARGO_CLIENT_MODE, 0, -1); - free(proto); - kv_context_t *context = kv_client_register(mid); - - /* open */ - kv_db_type_t db_type = KVDB_BWTREE; // XXX take that from argv - kv_database_t *db = kv_open(context, server_addr_str, "db/booger", db_type); - assert(db != NULL); - - /* put */ - int key = 10; - int val = 10; - printf("putting val with size %lu for key %d\n", sizeof(val), key); - ret = kv_put(db, &key, sizeof(key), &val, sizeof(val)); - assert(ret == HG_SUCCESS); - - /* get */ - int remote_val; - hg_size_t vsize = sizeof(remote_val); - ret = kv_get(db, &key, sizeof(key), &remote_val, &vsize); - std::cout << "kv_get returned size " << vsize << std::endl; - assert(ret == HG_SUCCESS); - if (val == remote_val) { - printf("put/get succeeded, key: %d in: %d out: %d\n", key, val, remote_val); - } - else { - printf("put/get failed, key: %d in: %d out: %d\n", key, val, remote_val); - } - - key = 20; - std::vector bulk_val(256, 10); - vsize = bulk_val.size() * sizeof(int32_t); - printf("putting bulk_val with size %lu for key %d\n", vsize, key); - ret = kv_put(db, &key, sizeof(key), bulk_val.data(), vsize); - assert(ret == HG_SUCCESS); - - std::vector data; - std::vector remote_bulk_val; - vsize = bulk_val.size() * sizeof(int32_t); - data.resize(vsize); - ret = kv_get(db, &key, sizeof(key), data.data(), &vsize); - std::cout << "kv_get returned size " << vsize << std::endl; - assert(ret == HG_SUCCESS); - std::cout << "resizing remote_bulk_val to " << vsize/sizeof(int32_t) << std::endl; - remote_bulk_val.resize(vsize/sizeof(int32_t)); - memcpy(remote_bulk_val.data(), data.data(), vsize); - printf("bulk_val size %lu, remote_bulk_val size %lu\n", bulk_val.size(), remote_bulk_val.size()); - if (bulk_val == remote_bulk_val) { - printf("bulk value put/get succeeded for key: %d\n", key); - } - else { - printf("bulk value put/get failed for key: %d\n", key); - std::cout << "bulk_val: " << bulk_val[0] << std::endl; - std::cout << "remote_bulk_val: " << remote_bulk_val[0] << std::endl; - } - - /* TODO: put more keys to better exercise listing */ - /* TODO: list only a subset of keys */ - /* listing all keys in DB */ - int i, start_key=0; - hg_size_t max_keys=10; - void **keys; - hg_size_t *sizes; - keys = (void **)calloc(max_keys, sizeof(void *)); - for (i=0; i< max_keys; i++) - keys[i] = calloc(1, sizeof(int)); - sizes = (hg_size_t *)calloc(max_keys, sizeof(*sizes)); - - ret = kv_list_keys(db, &start_key, sizeof(start_key), - keys, sizes, &max_keys); - for(int i=0; i< max_keys; i++) { - printf("found: %d of %d: %d (%zd)\n", i+1, max_keys, *(int *)keys[i], sizes[i]); - } - - bench_result_t *output; - output = kv_benchmark(db, 1000); - printf("inserts: %zd keys in %f seconds: %f Million-inserts per sec\n", - output->nkeys, output->insert_time, - output->nkeys/(output->insert_time*1000*1000) ); - printf("reads: %zd keys in %f seconds: %f Million-reads per sec\n", - output->nkeys, output->read_time, - output->nkeys/(output->read_time*1000*1000) ); - printf("overhead: %f seconds\n", output->overhead); - free(output); - - /* close */ - ret = kv_close(db); - assert(ret == HG_SUCCESS); - - /* signal server */ - ret = kv_client_signal_shutdown(db); - assert(ret == HG_SUCCESS); - - /* cleanup */ - ret = kv_client_deregister(context); - assert(ret == HG_SUCCESS); - - margo_finalize(mid); -} diff --git a/old-test/test-mpi-cci.conf b/old-test/test-mpi-cci.conf deleted file mode 100644 index 042583f..0000000 --- a/old-test/test-mpi-cci.conf +++ /dev/null @@ -1,4 +0,0 @@ -[ethernet] -# use this example for TCP -transport = tcp -interface = eth2 # switch this to eth2 or to an external hostname for non-localhost use diff --git a/old-test/test-mpi-group.cc b/old-test/test-mpi-group.cc deleted file mode 100644 index 7285ed2..0000000 --- a/old-test/test-mpi-group.cc +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright (c) 2017, Los Alamos National Security, LLC. - * All rights reserved. - * - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "sds-keyval-group.h" - -#include -#include -#include - -#include - -#define DIE_IF(cond_expr, err_fmt, ...) \ - do { \ - if (cond_expr) { \ - fprintf(stderr, "ERROR at %s:%d (" #cond_expr "): " \ - err_fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ - exit(1); \ - } \ - } while(0) - -// test-mpi-group addr_str num_servers [vsize] -// vsize is size of std::vector with default of 1 -int main(int argc, char *argv[]) -{ - int rank; - int nranks; - - assert(argc >= 3); - char *addr_str = argv[1]; - int num_servers = atoi(argv[2]); - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nranks); - - assert(nranks >= (num_servers+1)); // insist on at least 1 clients - - MPI_Comm clientComm, ssgComm; - - if (rank < num_servers) { - hg_size_t addr_str_sz = 128; - char server_addr_str[addr_str_sz]; - hg_addr_t server_addr; - hg_return_t hret; - const char *ssg_name = "test-server-group"; - - MPI_Comm_split(MPI_COMM_WORLD, 0, rank, &ssgComm); - MPI_Comm_split(MPI_COMM_WORLD, MPI_UNDEFINED, rank, &clientComm); - - // kv-server - kv_group_t *group = kvgroup_server_register(margo_init(addr_str, - MARGO_SERVER_MODE, 0, -1), - ssg_name, - ssgComm); - hret = margo_addr_self(group->mid, &server_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_self"); - - // get server address - hret = margo_addr_to_string(group->mid, server_addr_str, &addr_str_sz, server_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string"); - margo_addr_free(group->mid, server_addr); - - printf("server (rank %d): server addr_str: %s, group: %s\n", rank, server_addr_str, ssg_name); - - int server_rank; - MPI_Comm_rank(ssgComm, &server_rank); - - // broadcast (send) SSG ID to all clients - kvgroup_server_send_gid(group->gid, MPI_COMM_WORLD); - if (server_rank == 0) { - printf("server (rank %d): sent group\n", rank); - } - - // process requests until finalized - kvgroup_server_wait_for_shutdown(group); - - // now finish cleaning up - kvgroup_server_deregister(group); - printf("rank %d: server deregistered\n", rank); - - //kv_margo_finalize(mid); // already finalized in server's shutdown_handler - } - else { - hg_size_t addr_str_sz = 128; - char client_addr_str[addr_str_sz]; - hg_addr_t client_addr; - hg_return_t hret; - - MPI_Comm_split(MPI_COMM_WORLD, MPI_UNDEFINED, rank, &ssgComm); - MPI_Comm_split(MPI_COMM_WORLD, 1, rank, &clientComm); - - // broadcast (recv) SSG ID - ssg_group_id_t gid; - kvgroup_client_recv_gid(&gid, MPI_COMM_WORLD); - printf("client (rank %d): received group\n", rank); - - // kv-client - char *proto = kvgroup_protocol(gid); - kv_group_t *group= kvgroup_client_register(margo_init(proto, MARGO_CLIENT_MODE, 0, -1), gid); - free(proto); - hret = margo_addr_self(group->mid, &client_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_self"); - - // get client address - hret = margo_addr_to_string(group->mid, client_addr_str, &addr_str_sz, client_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string"); - margo_addr_free(group->mid, client_addr); - printf("client (rank %d): client addr_str: %s\n", rank, client_addr_str); - - // open specified "DB" (pass in the server's address) - const char *db = "db/minima_store"; - kv_db_type_t db_type = KVDB_BWTREE; - hret = kvgroup_open(group, (char*)db, db_type); - DIE_IF(hret != HG_SUCCESS, "kvgroup_open"); - - size_t vsize = 1; - if (argc == 4) { - vsize = atoi(argv[2]); - } - printf("client (rank %d): using vsize = %lu, dsize = %lu\n", rank, vsize, vsize*sizeof(int32_t)); - - // put - for (int i=1; i<1000; i++) { - int32_t key = 1000*rank + i; - std::vector put_data; - put_data.resize(vsize, key); - hg_size_t data_size = put_data.size()*sizeof(int32_t); // size in char (bytes) - - // create OID for key - size_t oid = boost::hash()(key); - hret = kvgroup_put(group, oid, (void*)&key, sizeof(key), - (void*)put_data.data(), data_size); - printf("(rank %d: put) key %d, size=%lu\n", rank, key, data_size); - DIE_IF(hret != HG_SUCCESS, "kv_put"); - } - - sleep(2); - - // get - for (int i=1; i<1000; i++) { - int32_t key = 1000*rank + i; - std::vector expected_get_data; - expected_get_data.resize(vsize, key); - std::vector get_data; - get_data.resize(vsize); - hg_size_t data_size = get_data.size()*sizeof(int32_t); // size in char (bytes) - printf("(rank %d: get) key %d, size=%lu\n", rank, key, data_size); - - // create OID for key - size_t oid = boost::hash()(key); - hret = kvgroup_get(group, oid, (void*)&key, sizeof(key), - (void*)get_data.data(), &data_size); - DIE_IF(hret != HG_SUCCESS, "kv_get"); - - get_data.resize(data_size/sizeof(int32_t)); // size in int32_t - if (expected_get_data == get_data) { - printf("(rank %d: put/get succeeded) key %d, actual size=%lu\n", rank, key, data_size); - } - else { - printf("(rank %d: put/get failed) key %d, actual size=%lu\n", rank, key, data_size); - } - } - - // close - hret = kvgroup_close(group); - DIE_IF(hret != HG_SUCCESS, "kv_close"); - - // once all clients are done with the close, one client can signal server - int client_rank; - MPI_Comm_rank(clientComm, &client_rank); - MPI_Barrier(clientComm); - if (client_rank==0) { - printf("rank %d: sending server a shutdown request\n", rank); - kvgroup_client_signal_shutdown(group); - } - - // now finish cleaning up - kvgroup_client_deregister(group); - printf("rank %d: client deregistered\n", rank); - - //kv_margo_finalize(mid); // already finalized in kv_client_deregister - } - - MPI_Finalize(); - - printf("rank %d: finalized\n", rank); - return 0; -} diff --git a/old-test/test-mpi-srun-hosts.conf b/old-test/test-mpi-srun-hosts.conf deleted file mode 100644 index 41d7a35..0000000 --- a/old-test/test-mpi-srun-hosts.conf +++ /dev/null @@ -1,2 +0,0 @@ -cn194 -cn323 diff --git a/old-test/test-mpi-srun-mp.conf b/old-test/test-mpi-srun-mp.conf deleted file mode 100644 index 541be96..0000000 --- a/old-test/test-mpi-srun-mp.conf +++ /dev/null @@ -1,2 +0,0 @@ -0 /home/dor/sds-keyval/test/test-server -1 /home/dor/sds-keyval/test/test-client tcp://192.168.100.49:52345 diff --git a/old-test/test-mpi.cc b/old-test/test-mpi.cc deleted file mode 100644 index b0dca50..0000000 --- a/old-test/test-mpi.cc +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2017, Los Alamos National Security, LLC. - * All rights reserved. - * - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "sds-keyval.h" - -#include -#include -#include - -#define DIE_IF(cond_expr, err_fmt, ...) \ - do { \ - if (cond_expr) { \ - fprintf(stderr, "ERROR at %s:%d (" #cond_expr "): " \ - err_fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \ - exit(1); \ - } \ - } while(0) - -int main(int argc, char *argv[]) -{ - int rank; - - assert(argc == 2); - char *addr_str = argv[1]; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - MPI_Comm clientComm; - - if (rank == 0) { - hg_size_t addr_str_sz = 128; - char server_addr_str[addr_str_sz]; - hg_addr_t server_addr; - hg_return_t hret; - - MPI_Comm_split(MPI_COMM_WORLD, MPI_UNDEFINED, rank, &clientComm); - - // kv-server - margo_instance_id mid = margo_init(addr_str, MARGO_SERVER_MODE, 0, -1); - kv_context_t *context = kv_server_register(mid); - - hret = margo_addr_self(mid, &server_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_self"); - - // get server address - hret = margo_addr_to_string(mid, server_addr_str, &addr_str_sz, server_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string"); - margo_addr_free(mid, server_addr); - - // broadcast (send) server address to all clients - printf("server (rank %d): server addr_str: %s\n", rank, server_addr_str); - MPI_Bcast(server_addr_str, 128, MPI_BYTE, 0, MPI_COMM_WORLD); - - // process requests until finalized - kv_server_wait_for_shutdown(context); - - // now finish cleaning up - kv_server_deregister(context); - printf("rank %d: server deregistered\n", rank); - - margo_finalize(mid); - } - else { - hg_size_t addr_str_sz = 128; - char server_addr_str[addr_str_sz]; - char client_addr_str[addr_str_sz]; - hg_addr_t client_addr; - hg_return_t hret; - kv_database_t * kv_db; - - MPI_Comm_split(MPI_COMM_WORLD, 1, rank, &clientComm); - - // broadcast (recv) server address - MPI_Bcast(server_addr_str, 128, MPI_BYTE, 0, MPI_COMM_WORLD); - printf("client (rank %d): server addr_str: %s\n", rank, server_addr_str); - - // kv-client - char *proto = kv_protocol(server_addr_str); - margo_instance_id mid = margo_init(proto, MARGO_CLIENT_MODE, 0, -1); - free(proto); - kv_context_t *context = kv_client_register(mid); - - hret = margo_addr_self(mid, &client_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_self"); - - // get client address - hret = margo_addr_to_string(mid, client_addr_str, &addr_str_sz, client_addr); - DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string"); - margo_addr_free(mid, client_addr); - printf("client (rank %d): client addr_str: %s\n", rank, client_addr_str); - - // open specified "DB" (pass in the server's address) - const char *db = "db/minima_store"; - kv_db_type_t db_type = KVDB_BWTREE; - kv_db = kv_open(context, server_addr_str, db, db_type); - DIE_IF(kv_db == NULL, "kv_open"); - - size_t vsize = 1; - if (argc == 3) { - vsize = atoi(argv[2]); - } - printf("client (rank %d): using vsize = %lu, dsize = %lu\n", rank, vsize, vsize*sizeof(int32_t)); - - // put - for (int i=1; i<1000; i++) { - int32_t key = 1000*rank + i; - std::vector put_data; - put_data.resize(vsize, key); - hg_size_t data_size = put_data.size()*sizeof(int32_t); // size in char (bytes) - - hret = kv_put(kv_db, (void*)&key, sizeof(key), - (void*)put_data.data(), data_size); - printf("(rank %d: put) key %d, size=%lu\n", rank, key, data_size); - DIE_IF(hret != HG_SUCCESS, "kv_put"); - } - - sleep(2); - - // get - for (int i=1; i<1000; i++) { - int32_t key = 1000*rank + i; - std::vector expected_get_data; - expected_get_data.resize(vsize, key); - std::vector get_data; - get_data.resize(vsize); - hg_size_t data_size = get_data.size()*sizeof(int32_t); // size in char (bytes) - printf("(rank %d: get) key %d, size=%lu\n", rank, key, data_size); - hret = kv_get(kv_db, (void*)&key, sizeof(key), - (void*)get_data.data(), &data_size); - DIE_IF(hret != HG_SUCCESS, "kv_get"); - - get_data.resize(data_size/sizeof(int32_t)); // size in int32_t - if (expected_get_data == get_data) { - printf("(rank %d: put/get succeeded) key %d, actual size=%lu\n", rank, key, data_size); - } - else { - printf("(rank %d: put/get failed) key %d, actual size=%lu\n", rank, key, data_size); - } - } - - // close - hret = kv_close(kv_db); - DIE_IF(hret != HG_SUCCESS, "kv_close"); - - // once all clients are done with the close, one client can signal server - MPI_Barrier(clientComm); - if (rank==1) { - printf("rank %d: sending server a shutdown request\n", rank); - kv_client_signal_shutdown(kv_db); - } - - // now finish cleaning up - kv_client_deregister(context); - printf("rank %d: client deregistered\n", rank); - - margo_finalize(mid); - } - - MPI_Finalize(); - - printf("rank %d: finalized\n", rank); - return 0; -} diff --git a/old-test/test-server.cc b/old-test/test-server.cc deleted file mode 100644 index 4e8c132..0000000 --- a/old-test/test-server.cc +++ /dev/null @@ -1,18 +0,0 @@ -#include "sds-keyval.h" -#include - -int main(int argc, char **argv) { - - assert(argc == 2); - - margo_instance_id mid = margo_init(argv[1] == NULL ? "ofi+tcp://" : argv[1], - MARGO_SERVER_MODE, 0, -1); - kv_context_t *context = kv_server_register(mid); - - hg_return_t ret; - ret = kv_server_wait_for_shutdown(context); - assert(ret == HG_SUCCESS); - - ret = kv_server_deregister(context); - assert(ret == HG_SUCCESS); -} -- 2.26.2