Skip to content

S
sds-keyval
Commit 20b130ca authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files
Options

removed old tests

parent b5544d67
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 625 deletions
old-test/bench-client.cc deleted 100644 → 0
View file @ b5544d67
#include <sds-keyval.h>
#include <assert.h>
#include <random>
#include <chrono>
void RandomInsertSpeedTest(kv_database_t *db,
size_t key_num, bench_result_t *results)
{
hg_return_t ret;
std::random_device r{};
std::default_random_engine e1(r());
std::uniform_int_distribution<int32_t> uniform_dist(0, key_num - 1);
std::chrono::time_point<std::chrono::system_clock> start, end;
// We loop for keynum * 2 because in average half of the insertion
// will hit an empty slot
start = std::chrono::system_clock::now();
for(size_t i = 0;i < key_num * 2;i++) {
int32_t key = uniform_dist(e1);
ret = kv_put(db, &key, sizeof(key), &key, sizeof(key));
assert(ret == HG_SUCCESS);
}
end = std::chrono::system_clock::now();
std::chrono::duration<double> elapsed_seconds = end - start;
results->nkeys = key_num *2;
results->insert_time = elapsed_seconds.count();
// Then test random read after random insert
int32_t value;
hg_size_t vsize;
start = std::chrono::system_clock::now();
for(size_t i = 0;i < key_num * 2;i++) {
int32_t key = uniform_dist(e1);
vsize = sizeof(value);
ret = kv_get(db, &key, sizeof(key), &value, &vsize);
// key might not be in store due to randomness
// HG_OTHER_ERROR is basically a "key not found" return code
assert(ret == HG_SUCCESS || ret == HG_OTHER_ERROR);
}
end = std::chrono::system_clock::now();
elapsed_seconds = end - start;
results->read_time = elapsed_seconds.count();
results->overhead = 0;
return;
}
void print_results(bench_result_t *r)
{
printf("inserts: %zd keys in %f seconds: %f Million-inserts per sec: %f usec per insert\n",
r->nkeys, r->insert_time, r->nkeys/(r->insert_time*1000*1000),
(r->insert_time*1000*1000)/r->nkeys);
printf("reads: %zd keys in %f seconds: %f Million-reads per sec: %f usec per read\n",
r->nkeys, r->read_time, r->nkeys/(r->read_time*1000*1000),
(r->read_time*1000*1000)/r->nkeys);
}
int main(int argc, char **argv)
{
hg_return_t ret;
bench_result_t rpc;
kv_context_t *context;
kv_database_t *db;
assert(argc == 3);
size_t items = atoi(argv[1]);
char *server_addr_str = argv[2];
char *proto = kv_protocol(server_addr_str);
margo_instance_id mid = margo_init(proto, MARGO_CLIENT_MODE, 0, -1);
free(proto);
context = kv_client_register(mid);
kv_db_type_t db_type = KVDB_BWTREE; // XXX make that an argument from argv
db = kv_open(context, server_addr_str, "db/testdb", db_type);
assert(db != NULL);
RandomInsertSpeedTest(db, items, &rpc);
print_results(&rpc);
bench_result_t *server;
server = kv_benchmark(db, items);
print_results(server);
free(server);
/* close */
ret = kv_close(db);
assert(ret == HG_SUCCESS);
/* signal server */
ret = kv_client_signal_shutdown(db);
assert(ret == HG_SUCCESS);
/* cleanup */
ret = kv_client_deregister(context);
assert(ret == HG_SUCCESS);
margo_finalize(mid);
}
old-test/test-cci.conf deleted 100644 → 0
View file @ b5544d67
[ethernet]
# use this example for TCP
transport = tcp
interface = eth4 # switch this to eth2 or to an external hostname for non-localhost use
old-test/test-client.cc deleted 100644 → 0
View file @ b5544d67
#include "sds-keyval.h"
#include <unistd.h>
#include <assert.h>
#include <vector>
#include <iostream>
int main(int argc, char **argv) {
hg_return_t ret;
assert(argc == 2);
char *server_addr_str = argv[1];
char *proto = kv_protocol(server_addr_str);
margo_instance_id mid = margo_init(proto, MARGO_CLIENT_MODE, 0, -1);
free(proto);
kv_context_t *context = kv_client_register(mid);
/* open */
kv_db_type_t db_type = KVDB_BWTREE; // XXX take that from argv
kv_database_t *db = kv_open(context, server_addr_str, "db/booger", db_type);
assert(db != NULL);
/* put */
int key = 10;
int val = 10;
printf("putting val with size %lu for key %d\n", sizeof(val), key);
ret = kv_put(db, &key, sizeof(key), &val, sizeof(val));
assert(ret == HG_SUCCESS);
/* get */
int remote_val;
hg_size_t vsize = sizeof(remote_val);
ret = kv_get(db, &key, sizeof(key), &remote_val, &vsize);
std::cout << "kv_get returned size " << vsize << std::endl;
assert(ret == HG_SUCCESS);
if (val == remote_val) {
printf("put/get succeeded, key: %d in: %d out: %d\n", key, val, remote_val);
}
else {
printf("put/get failed, key: %d in: %d out: %d\n", key, val, remote_val);
}
key = 20;
std::vector<int32_t> bulk_val(256, 10);
vsize = bulk_val.size() * sizeof(int32_t);
printf("putting bulk_val with size %lu for key %d\n", vsize, key);
ret = kv_put(db, &key, sizeof(key), bulk_val.data(), vsize);
assert(ret == HG_SUCCESS);
std::vector<char> data;
std::vector<int32_t> remote_bulk_val;
vsize = bulk_val.size() * sizeof(int32_t);
data.resize(vsize);
ret = kv_get(db, &key, sizeof(key), data.data(), &vsize);
std::cout << "kv_get returned size " << vsize << std::endl;
assert(ret == HG_SUCCESS);
std::cout << "resizing remote_bulk_val to " << vsize/sizeof(int32_t) << std::endl;
remote_bulk_val.resize(vsize/sizeof(int32_t));
memcpy(remote_bulk_val.data(), data.data(), vsize);
printf("bulk_val size %lu, remote_bulk_val size %lu\n", bulk_val.size(), remote_bulk_val.size());
if (bulk_val == remote_bulk_val) {
printf("bulk value put/get succeeded for key: %d\n", key);
}
else {
printf("bulk value put/get failed for key: %d\n", key);
std::cout << "bulk_val: " << bulk_val[0] << std::endl;
std::cout << "remote_bulk_val: " << remote_bulk_val[0] << std::endl;
}
/* TODO: put more keys to better exercise listing */
/* TODO: list only a subset of keys */
/* listing all keys in DB */
int i, start_key=0;
hg_size_t max_keys=10;
void **keys;
hg_size_t *sizes;
keys = (void **)calloc(max_keys, sizeof(void *));
for (i=0; i< max_keys; i++)
keys[i] = calloc(1, sizeof(int));
sizes = (hg_size_t *)calloc(max_keys, sizeof(*sizes));
ret = kv_list_keys(db, &start_key, sizeof(start_key),
keys, sizes, &max_keys);
for(int i=0; i< max_keys; i++) {
printf("found: %d of %d: %d (%zd)\n", i+1, max_keys, *(int *)keys[i], sizes[i]);
}
bench_result_t *output;
output = kv_benchmark(db, 1000);
printf("inserts: %zd keys in %f seconds: %f Million-inserts per sec\n",
output->nkeys, output->insert_time,
output->nkeys/(output->insert_time*1000*1000) );
printf("reads: %zd keys in %f seconds: %f Million-reads per sec\n",
output->nkeys, output->read_time,
output->nkeys/(output->read_time*1000*1000) );
printf("overhead: %f seconds\n", output->overhead);
free(output);
/* close */
ret = kv_close(db);
assert(ret == HG_SUCCESS);
/* signal server */
ret = kv_client_signal_shutdown(db);
assert(ret == HG_SUCCESS);
/* cleanup */
ret = kv_client_deregister(context);
assert(ret == HG_SUCCESS);
margo_finalize(mid);
}
old-test/test-mpi-cci.conf deleted 100644 → 0
View file @ b5544d67
[ethernet]
# use this example for TCP
transport = tcp
interface = eth2 # switch this to eth2 or to an external hostname for non-localhost use
old-test/test-mpi-group.cc deleted 100644 → 0
View file @ b5544d67
/*
* Copyright (c) 2017, Los Alamos National Security, LLC.
* All rights reserved.
*
*/
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <mpi.h>
#include <margo.h>
#include <mercury.h>
#include <abt.h>
#include "sds-keyval-group.h"
#include <vector>
#include <stdlib.h>
#include <time.h>
#include <boost/functional/hash.hpp>
#define DIE_IF(cond_expr, err_fmt, ...) \
do { \
if (cond_expr) { \
fprintf(stderr, "ERROR at %s:%d (" #cond_expr "): " \
err_fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \
exit(1); \
} \
} while(0)
// test-mpi-group addr_str num_servers [vsize]
// vsize is size of std::vector<int32_t> with default of 1
int main(int argc, char *argv[])
{
int rank;
int nranks;
assert(argc >= 3);
char *addr_str = argv[1];
int num_servers = atoi(argv[2]);
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nranks);
assert(nranks >= (num_servers+1)); // insist on at least 1 clients
MPI_Comm clientComm, ssgComm;
if (rank < num_servers) {
hg_size_t addr_str_sz = 128;
char server_addr_str[addr_str_sz];
hg_addr_t server_addr;
hg_return_t hret;
const char *ssg_name = "test-server-group";
MPI_Comm_split(MPI_COMM_WORLD, 0, rank, &ssgComm);
MPI_Comm_split(MPI_COMM_WORLD, MPI_UNDEFINED, rank, &clientComm);
// kv-server
kv_group_t *group = kvgroup_server_register(margo_init(addr_str,
MARGO_SERVER_MODE, 0, -1),
ssg_name,
ssgComm);
hret = margo_addr_self(group->mid, &server_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_self");
// get server address
hret = margo_addr_to_string(group->mid, server_addr_str, &addr_str_sz, server_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string");
margo_addr_free(group->mid, server_addr);
printf("server (rank %d): server addr_str: %s, group: %s\n", rank, server_addr_str, ssg_name);
int server_rank;
MPI_Comm_rank(ssgComm, &server_rank);
// broadcast (send) SSG ID to all clients
kvgroup_server_send_gid(group->gid, MPI_COMM_WORLD);
if (server_rank == 0) {
printf("server (rank %d): sent group\n", rank);
}
// process requests until finalized
kvgroup_server_wait_for_shutdown(group);
// now finish cleaning up
kvgroup_server_deregister(group);
printf("rank %d: server deregistered\n", rank);
//kv_margo_finalize(mid); // already finalized in server's shutdown_handler
}
else {
hg_size_t addr_str_sz = 128;
char client_addr_str[addr_str_sz];
hg_addr_t client_addr;
hg_return_t hret;
MPI_Comm_split(MPI_COMM_WORLD, MPI_UNDEFINED, rank, &ssgComm);
MPI_Comm_split(MPI_COMM_WORLD, 1, rank, &clientComm);
// broadcast (recv) SSG ID
ssg_group_id_t gid;
kvgroup_client_recv_gid(&gid, MPI_COMM_WORLD);
printf("client (rank %d): received group\n", rank);
// kv-client
char *proto = kvgroup_protocol(gid);
kv_group_t *group= kvgroup_client_register(margo_init(proto, MARGO_CLIENT_MODE, 0, -1), gid);
free(proto);
hret = margo_addr_self(group->mid, &client_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_self");
// get client address
hret = margo_addr_to_string(group->mid, client_addr_str, &addr_str_sz, client_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string");
margo_addr_free(group->mid, client_addr);
printf("client (rank %d): client addr_str: %s\n", rank, client_addr_str);
// open specified "DB" (pass in the server's address)
const char *db = "db/minima_store";
kv_db_type_t db_type = KVDB_BWTREE;
hret = kvgroup_open(group, (char*)db, db_type);
DIE_IF(hret != HG_SUCCESS, "kvgroup_open");
size_t vsize = 1;
if (argc == 4) {
vsize = atoi(argv[2]);
}
printf("client (rank %d): using vsize = %lu, dsize = %lu\n", rank, vsize, vsize*sizeof(int32_t));
// put
for (int i=1; i<1000; i++) {
int32_t key = 1000*rank + i;
std::vector<int32_t> put_data;
put_data.resize(vsize, key);
hg_size_t data_size = put_data.size()*sizeof(int32_t); // size in char (bytes)
// create OID for key
size_t oid = boost::hash<int32_t>()(key);
hret = kvgroup_put(group, oid, (void*)&key, sizeof(key),
(void*)put_data.data(), data_size);
printf("(rank %d: put) key %d, size=%lu\n", rank, key, data_size);
DIE_IF(hret != HG_SUCCESS, "kv_put");
}
sleep(2);
// get
for (int i=1; i<1000; i++) {
int32_t key = 1000*rank + i;
std::vector<int32_t> expected_get_data;
expected_get_data.resize(vsize, key);
std::vector<int32_t> get_data;
get_data.resize(vsize);
hg_size_t data_size = get_data.size()*sizeof(int32_t); // size in char (bytes)
printf("(rank %d: get) key %d, size=%lu\n", rank, key, data_size);
// create OID for key
size_t oid = boost::hash<int32_t>()(key);
hret = kvgroup_get(group, oid, (void*)&key, sizeof(key),
(void*)get_data.data(), &data_size);
DIE_IF(hret != HG_SUCCESS, "kv_get");
get_data.resize(data_size/sizeof(int32_t)); // size in int32_t
if (expected_get_data == get_data) {
printf("(rank %d: put/get succeeded) key %d, actual size=%lu\n", rank, key, data_size);
}
else {
printf("(rank %d: put/get failed) key %d, actual size=%lu\n", rank, key, data_size);
}
}
// close
hret = kvgroup_close(group);
DIE_IF(hret != HG_SUCCESS, "kv_close");
// once all clients are done with the close, one client can signal server
int client_rank;
MPI_Comm_rank(clientComm, &client_rank);
MPI_Barrier(clientComm);
if (client_rank==0) {
printf("rank %d: sending server a shutdown request\n", rank);
kvgroup_client_signal_shutdown(group);
}
// now finish cleaning up
kvgroup_client_deregister(group);
printf("rank %d: client deregistered\n", rank);
//kv_margo_finalize(mid); // already finalized in kv_client_deregister
}
MPI_Finalize();
printf("rank %d: finalized\n", rank);
return 0;
}
old-test/test-mpi-srun-mp.conf deleted 100644 → 0
View file @ b5544d67
0 /home/dor/sds-keyval/test/test-server
1 /home/dor/sds-keyval/test/test-client tcp://192.168.100.49:52345
old-test/test-mpi.cc deleted 100644 → 0
View file @ b5544d67
/*
* Copyright (c) 2017, Los Alamos National Security, LLC.
* All rights reserved.
*
*/
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <mpi.h>
#include <margo.h>
#include <mercury.h>
#include <abt.h>
#include "sds-keyval.h"
#include <vector>
#include <stdlib.h>
#include <time.h>
#define DIE_IF(cond_expr, err_fmt, ...) \
do { \
if (cond_expr) { \
fprintf(stderr, "ERROR at %s:%d (" #cond_expr "): " \
err_fmt "\n", __FILE__, __LINE__, ##__VA_ARGS__); \
exit(1); \
} \
} while(0)
int main(int argc, char *argv[])
{
int rank;
assert(argc == 2);
char *addr_str = argv[1];
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm clientComm;
if (rank == 0) {
hg_size_t addr_str_sz = 128;
char server_addr_str[addr_str_sz];
hg_addr_t server_addr;
hg_return_t hret;
MPI_Comm_split(MPI_COMM_WORLD, MPI_UNDEFINED, rank, &clientComm);
// kv-server
margo_instance_id mid = margo_init(addr_str, MARGO_SERVER_MODE, 0, -1);
kv_context_t *context = kv_server_register(mid);
hret = margo_addr_self(mid, &server_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_self");
// get server address
hret = margo_addr_to_string(mid, server_addr_str, &addr_str_sz, server_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string");
margo_addr_free(mid, server_addr);
// broadcast (send) server address to all clients
printf("server (rank %d): server addr_str: %s\n", rank, server_addr_str);
MPI_Bcast(server_addr_str, 128, MPI_BYTE, 0, MPI_COMM_WORLD);
// process requests until finalized
kv_server_wait_for_shutdown(context);
// now finish cleaning up
kv_server_deregister(context);
printf("rank %d: server deregistered\n", rank);
margo_finalize(mid);
}
else {
hg_size_t addr_str_sz = 128;
char server_addr_str[addr_str_sz];
char client_addr_str[addr_str_sz];
hg_addr_t client_addr;
hg_return_t hret;
kv_database_t * kv_db;
MPI_Comm_split(MPI_COMM_WORLD, 1, rank, &clientComm);
// broadcast (recv) server address
MPI_Bcast(server_addr_str, 128, MPI_BYTE, 0, MPI_COMM_WORLD);
printf("client (rank %d): server addr_str: %s\n", rank, server_addr_str);
// kv-client
char *proto = kv_protocol(server_addr_str);
margo_instance_id mid = margo_init(proto, MARGO_CLIENT_MODE, 0, -1);
free(proto);
kv_context_t *context = kv_client_register(mid);
hret = margo_addr_self(mid, &client_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_self");
// get client address
hret = margo_addr_to_string(mid, client_addr_str, &addr_str_sz, client_addr);
DIE_IF(hret != HG_SUCCESS, "margo_addr_to_string");
margo_addr_free(mid, client_addr);
printf("client (rank %d): client addr_str: %s\n", rank, client_addr_str);
// open specified "DB" (pass in the server's address)
const char *db = "db/minima_store";
kv_db_type_t db_type = KVDB_BWTREE;
kv_db = kv_open(context, server_addr_str, db, db_type);
DIE_IF(kv_db == NULL, "kv_open");
size_t vsize = 1;
if (argc == 3) {
vsize = atoi(argv[2]);
}
printf("client (rank %d): using vsize = %lu, dsize = %lu\n", rank, vsize, vsize*sizeof(int32_t));
// put
for (int i=1; i<1000; i++) {
int32_t key = 1000*rank + i;
std::vector<int32_t> put_data;
put_data.resize(vsize, key);
hg_size_t data_size = put_data.size()*sizeof(int32_t); // size in char (bytes)
hret = kv_put(kv_db, (void*)&key, sizeof(key),
(void*)put_data.data(), data_size);
printf("(rank %d: put) key %d, size=%lu\n", rank, key, data_size);
DIE_IF(hret != HG_SUCCESS, "kv_put");
}
sleep(2);
// get
for (int i=1; i<1000; i++) {
int32_t key = 1000*rank + i;
std::vector<int32_t> expected_get_data;
expected_get_data.resize(vsize, key);
std::vector<int32_t> get_data;
get_data.resize(vsize);
hg_size_t data_size = get_data.size()*sizeof(int32_t); // size in char (bytes)
printf("(rank %d: get) key %d, size=%lu\n", rank, key, data_size);
hret = kv_get(kv_db, (void*)&key, sizeof(key),
(void*)get_data.data(), &data_size);
DIE_IF(hret != HG_SUCCESS, "kv_get");
get_data.resize(data_size/sizeof(int32_t)); // size in int32_t
if (expected_get_data == get_data) {
printf("(rank %d: put/get succeeded) key %d, actual size=%lu\n", rank, key, data_size);
}
else {
printf("(rank %d: put/get failed) key %d, actual size=%lu\n", rank, key, data_size);
}
}
// close
hret = kv_close(kv_db);
DIE_IF(hret != HG_SUCCESS, "kv_close");
// once all clients are done with the close, one client can signal server
MPI_Barrier(clientComm);
if (rank==1) {
printf("rank %d: sending server a shutdown request\n", rank);
kv_client_signal_shutdown(kv_db);
}
// now finish cleaning up
kv_client_deregister(context);
printf("rank %d: client deregistered\n", rank);
margo_finalize(mid);
}
MPI_Finalize();
printf("rank %d: finalized\n", rank);
return 0;
}
old-test/test-server.cc deleted 100644 → 0
View file @ b5544d67
#include "sds-keyval.h"
#include <assert.h>
int main(int argc, char **argv) {
assert(argc == 2);
margo_instance_id mid = margo_init(argv[1] == NULL ? "ofi+tcp://" : argv[1],
MARGO_SERVER_MODE, 0, -1);
kv_context_t *context = kv_server_register(mid);
hg_return_t ret;
ret = kv_server_wait_for_shutdown(context);
assert(ret == HG_SUCCESS);
ret = kv_server_deregister(context);
assert(ret == HG_SUCCESS);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment