Commit a2de1206 authored by David Rich's avatar David Rich Committed by Rob Latham

Reworked/simplified datastore. Added support for BwTree and LevelDB. Both tested with test-mpi app.

parent d1fb9292
ACLOCAL_AMFLAGS="-Im4"
bin_PROGRAMS = test/bench-client
test_bench_client_SOURCES = test/bench-client.cc
test_bench_client_CPPFLAGS = ${CPPFLAGS} -I${srcdir}/src
test_bench_client_LDADD = -lkvclient
lib_LTLIBRARIES = libkvclient.la \
libkvserver.la
libkvclient_la_SOURCES = src/kv-client.c
libkvserver_la_SOURCES = src/kv-server.cc \
src/datastore.cc \
src/BwTree/src/bwtree.cpp
libkvserver_la_CXXFLAGS = -pthread -std=c++11 -g -Wall -mcx16 -Wno-invalid-offsetof
libkvserver_la_CPPFLAGS = -I${srcdir}/src/BwTree/src
libkvserver_la_CPPFLAGS = ${CPPFLAGS} -I${srcdir}/src/BwTree/src
include_HEADERS = src/sds-keyval.h
noinst_HEADERS = src/BwTree/src/bwtree.h \
......@@ -32,16 +38,21 @@ TESTS = test/test-client \
test/test-mpi
test_test_client_CPPFLAGS = ${CPPFLAGS} -I${srcdir}/src
test_test_client_LDADD = ${LIBS} -lkvclient
test_test_client_LDADD = -lkvclient
test_test_server_CPPFLAGS = ${CPPFLAGS} -I${srcdir}/src
test_test_server_LDADD = ${LIBS} -lkvserver -lstdc++
test_test_server_SOURCES = test/test-server.cc
test_test_server_CXXFLAGS = -pthread -std=c++11 -g -Wall -mcx16 -Wno-invalid-offsetof
test_test_server_CPPFLAGS = ${CPPFLAGS} -I${srcdir}/src -I${srcdir}/src/BwTree/src
test_test_server_LDADD = -lkvserver -lleveldb -lsnappy
test_test_mpi_SOURCES = test/test-mpi.cc
test_test_mpi_CPPFLAGS = ${CPPFLAGS} -I${srcdir}/src
test_test_mpi_LDADD = ${LIBS} -lkvclient -lkvserver
test_test_mpi_CXXFLAGS = -pthread -std=c++11 -g -Wall -mcx16 -Wno-invalid-offsetof
test_test_mpi_CPPFLAGS = ${CPPFLAGS} -I${srcdir}/src -I${srcdir}/src/BwTree/src
test_test_mpi_LDADD = -lkvclient -lkvserver -lleveldb -lsnappy
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = maint/kv-server.pc \
maint/kv-client.pc
// Copyright (c) 2017, Los Alamos National Security, LLC.
// All rights reserved.
#include "bwtree.h"
#include <vector>
using namespace wangziqi2013::bwtree;
#ifndef datastore_cc
#define datastore_cc
template <typename KeyType, typename ValueType> class AbstractDataStore {
public:
virtual void createDatabase(std::string homeDir, std::string baseName)=0;
virtual bool put(const KeyType &key, ValueType &data)=0;
virtual bool get(const KeyType &key, ValueType &data)=0;
virtual bool get(const KeyType &key, std::vector<ValueType> &data)=0;
};
template <typename KeyType,
typename ValueType,
typename KeyComparator = std::less<KeyType>,
typename KeyEqualityChecker = std::equal_to<KeyType>,
typename KeyHashFunc = std::hash<KeyType>,
typename ValueEqualityChecker = std::equal_to<ValueType>,
typename ValueHashFunc = std::hash<ValueType>>
class BwTreeDataStore : public AbstractDataStore<KeyType, ValueType> {
public:
BwTreeDataStore(Duplicates duplicates, bool eraseOnGet, bool debug) {
_duplicates = duplicates;
_eraseOnGet = eraseOnGet;
_debug = debug;
_tree = NULL;
};
#include "datastore.h"
AbstractDataStore::AbstractDataStore() {
_duplicates = Duplicates::IGNORE;
_eraseOnGet = false;
_debug = false;
};
AbstractDataStore::AbstractDataStore(Duplicates duplicates, bool eraseOnGet, bool debug) {
_duplicates = duplicates;
_eraseOnGet = eraseOnGet;
_debug = debug;
};
AbstractDataStore::~AbstractDataStore()
{};
BwTreeDataStore::BwTreeDataStore() :
AbstractDataStore() {
_tree = NULL;
};
BwTreeDataStore::BwTreeDataStore(Duplicates duplicates, bool eraseOnGet, bool debug) :
AbstractDataStore(duplicates, eraseOnGet, debug) {
_tree = NULL;
};
virtual ~BwTreeDataStore() {
delete _tree;
};
virtual void createDatabase(std::string homeDir, std::string baseName) {
_tree = new BwTree<KeyType, ValueType,
KeyComparator, KeyEqualityChecker, KeyHashFunc,
ValueEqualityChecker, ValueHashFunc>();
if (_debug) {
_tree->SetDebugLogging(1);
}
else {
_tree->SetDebugLogging(0);
}
_tree->UpdateThreadLocal(1);
_tree->AssignGCID(0);
};
BwTreeDataStore::~BwTreeDataStore() {
// deleting BwTree can cause core dump
delete _tree;
};
void BwTreeDataStore::createDatabase(std::string db_name) {
_tree = new BwTree<kv_key_t, ds_bulk_t, std::less<kv_key_t>,
std::equal_to<kv_key_t>, std::hash<kv_key_t>,
my_equal_to/*ds_bulk_t*/, my_hash/*ds_bulk_t*/>();
if (_debug) {
_tree->SetDebugLogging(1);
}
else {
_tree->SetDebugLogging(0);
}
_tree->UpdateThreadLocal(1);
_tree->AssignGCID(0);
};
virtual bool put(const KeyType &key, ValueType &data) {
std::vector<ValueType> values;
bool success = false;
bool BwTreeDataStore::put(const kv_key_t &key, ds_bulk_t &data) {
std::vector<ds_bulk_t> values;
bool success = false;
_tree->GetValue(key, values);
bool duplicate_key = (values.size() != 0);
if (duplicate_key) {
if (_duplicates == Duplicates::IGNORE) {
success = true;
}
else { // Duplicates::ALLOW
success = _tree->Insert(key, data);
}
_tree->GetValue(key, values);
bool duplicate_key = (values.size() != 0);
if (duplicate_key) {
if (_duplicates == Duplicates::IGNORE) {
// silently ignore
success = true;
}
else {
else { // Duplicates::ALLOW (default)
success = _tree->Insert(key, data);
}
}
else {
success = _tree->Insert(key, data);
}
return success;
};
return success;
};
virtual bool get(const KeyType &key, ValueType &data) {
std::vector<ValueType> values;
bool success = false;
bool BwTreeDataStore::get(const kv_key_t &key, ds_bulk_t &data) {
std::vector<ds_bulk_t> values;
bool success = false;
_tree->GetValue(key, values);
if (values.size() == 1) {
data = values.front();
_tree->GetValue(key, values);
if (values.size() == 1) {
data = values.front();
success = true;
}
else if (values.size() > 1) {
// this should only happen if duplicates are allowed
if (_duplicates == Duplicates::ALLOW) {
data = values.front(); // caller is asking for just 1
success = true;
}
else if (values.size() > 1) {
// this should only happen if duplicates are allowed
if (_duplicates == Duplicates::ALLOW) {
data = values.front(); // caller is asking for just 1
success = true;
}
}
return success;
};
bool BwTreeDataStore::get(const kv_key_t &key, std::vector<ds_bulk_t> &data) {
std::vector<ds_bulk_t> values;
bool success = false;
_tree->GetValue(key, values);
if (values.size() > 1) {
// this should only happen if duplicates are allowed
if (_duplicates == Duplicates::ALLOW) {
data = values;
success = true;
}
}
else {
data = values;
success = true;
}
return success;
};
return success;
};
virtual bool get(const KeyType &key, std::vector<ValueType> &data) {
std::vector<ValueType> values;
bool success = false;
LevelDBDataStore::LevelDBDataStore() :
AbstractDataStore() {
_dbm = NULL;
};
_tree->GetValue(key, values);
if (values.size() > 1) {
// this should only happen if duplicates are allowed
if (_duplicates == Duplicates::ALLOW) {
data = values;
success = true;
}
LevelDBDataStore::LevelDBDataStore(Duplicates duplicates, bool eraseOnGet, bool debug) :
AbstractDataStore(duplicates, eraseOnGet, debug) {
_dbm = NULL;
};
std::string LevelDBDataStore::key2string(const kv_key_t &key) {
kv_key_t k = key; // grab a copy to work with
char *c = reinterpret_cast<char *>(&k);
std::string keystr(c, sizeof(k));
return keystr;
};
kv_key_t LevelDBDataStore::string2key(std::string &keystr) {
kv_key_t *key = reinterpret_cast<kv_key_t*>(&(keystr[0]));
return *key;
};
LevelDBDataStore::~LevelDBDataStore() {
// deleting LevelDB can cause core dump
delete _dbm;
};
void LevelDBDataStore::createDatabase(std::string db_name) {
leveldb::Options options;
leveldb::Status status;
// db_name assumed to include the full path (e.g. /var/data/db.dat)
options.create_if_missing = true;
status = leveldb::DB::Open(options, db_name, &_dbm);
if (!status.ok()) {
// error
std::cerr << "LevelDBDataStore::createDatabase: LevelDB error on Open = " << status.ToString() << std::endl;
}
assert(status.ok()); // fall over
// debugging support?
};
bool LevelDBDataStore::put(const kv_key_t &key, ds_bulk_t &data) {
leveldb::Status status;
bool success = false;
std::string value;
status = _dbm->Get(leveldb::ReadOptions(), key2string(key), &value);
bool duplicate_key = false;
if (status.ok()) {
duplicate_key = true;
}
else if (!status.IsNotFound()) {
std::cerr << "LevelDBDataStore::put: LevelDB error on Get = " << status.ToString() << std::endl;
return false; // give up and return
}
bool insert_key = true;
if (duplicate_key) {
insert_key = false;
if (_duplicates == Duplicates::IGNORE) {
// silently ignore
success = true;
}
else {
data = values;
success = true;
std::cerr << "LevelDBDataStore::put: duplicate key " << key
<< ", duplicates not supported" << std::endl;
}
}
return success;
};
if (insert_key) {
std::string datastr(data.begin(), data.end());
status = _dbm->Put(leveldb::WriteOptions(), key2string(key), datastr);
if (status.ok()) {
success = true;
}
else {
std::cerr << "LevelDBDataStore::put: LevelDB error on Put = " << status.ToString() << std::endl;
}
}
protected:
BwTree<KeyType, ValueType,
KeyComparator, KeyEqualityChecker, KeyHashFunc,
ValueEqualityChecker, ValueHashFunc> *_tree = NULL;
Duplicates _duplicates;
bool _eraseOnGet;
bool _debug;
return success;
};
#endif // datastore_cc
bool LevelDBDataStore::get(const kv_key_t &key, ds_bulk_t &data) {
leveldb::Status status;
bool success = false;
data.clear();
std::string value;
status = _dbm->Get(leveldb::ReadOptions(), key2string(key), &value);
if (status.ok()) {
data = ds_bulk_t(value.begin(), value.end());
success = true;
}
else if (!status.IsNotFound()) {
std::cerr << "LevelDBDataStore::get: LevelDB error on Get = " << status.ToString() << std::endl;
}
return success;
};
bool LevelDBDataStore::get(const kv_key_t &key, std::vector<ds_bulk_t> &data) {
bool success = false;
data.clear();
ds_bulk_t value;
if (get(key, value)) {
data.push_back(value);
success = true;
}
return success;
};
// Copyright (c) 2017, Los Alamos National Security, LLC.
// All rights reserved.
#include "bwtree.h"
#include "sds-keyval.h"
#include <boost/functional/hash.hpp>
#include <vector>
#include <leveldb/db.h>
#include <db_cxx.h>
#include <dbstl_map.h>
using namespace wangziqi2013::bwtree;
#ifndef datastore_h
#define datastore_h
enum class Duplicates : int {ALLOW, IGNORE};
#endif // datastore_h
// implementation is std::vector<char> specific
// typedef is for convenience
typedef std::vector<char> ds_bulk_t;
struct my_hash {
size_t operator()(const ds_bulk_t &v) const {
size_t hash = 0;
boost::hash_range(hash, v.begin(), v.end());
return hash;
}
};
struct my_equal_to {
bool operator()(const ds_bulk_t &v1, const ds_bulk_t &v2) const {
return (v1 == v2);
}
};
#include "datastore.cc"
class AbstractDataStore {
public:
AbstractDataStore();
AbstractDataStore(Duplicates duplicates, bool eraseOnGet, bool debug);
virtual ~AbstractDataStore();
virtual void createDatabase(std::string db_name)=0;
virtual bool put(const kv_key_t &key, ds_bulk_t &data)=0;
virtual bool get(const kv_key_t &key, ds_bulk_t &data)=0;
virtual bool get(const kv_key_t &key, std::vector<ds_bulk_t> &data)=0;
protected:
Duplicates _duplicates;
bool _eraseOnGet;
bool _debug;
};
class BwTreeDataStore : public AbstractDataStore {
public:
BwTreeDataStore();
BwTreeDataStore(Duplicates duplicates, bool eraseOnGet, bool debug);
virtual ~BwTreeDataStore();
virtual void createDatabase(std::string db_name);
virtual bool put(const kv_key_t &key, ds_bulk_t &data);
virtual bool get(const kv_key_t &key, ds_bulk_t &data);
virtual bool get(const kv_key_t &key, std::vector<ds_bulk_t> &data);
protected:
BwTree<kv_key_t, ds_bulk_t, std::less<kv_key_t>,
std::equal_to<kv_key_t>, std::hash<kv_key_t>,
my_equal_to/*ds_bulk_t*/, my_hash/*ds_bulk_t*/> *_tree = NULL;
};
// may want to implement some caching for persistent stores like LevelDB
class LevelDBDataStore : public AbstractDataStore {
public:
LevelDBDataStore();
LevelDBDataStore(Duplicates duplicates, bool eraseOnGet, bool debug);
virtual ~LevelDBDataStore();
virtual void createDatabase(std::string db_name);
virtual bool put(const kv_key_t &key, ds_bulk_t &data);
virtual bool get(const kv_key_t &key, ds_bulk_t &data);
virtual bool get(const kv_key_t &key, std::vector<ds_bulk_t> &data);
protected:
leveldb::DB *_dbm = NULL;
Duplicates _duplicates;
bool _eraseOnGet;
bool _debug;
private:
std::string key2string(const kv_key_t &key);
kv_key_t string2key(std::string &keystr);
};
#endif // datastore_h
This diff is collapsed.
This diff is collapsed.
#include <stdint.h>
#include <mercury.h>
#include <mercury_macros.h>
#include <mercury_proc_string.h>
......@@ -6,25 +8,32 @@
#include <abt.h>
#include <abt-snoozer.h>
#ifndef sds_keyval_h
#define sds_keyval_h
#if defined(__cplusplus)
extern "C" {
#endif
#if 0
sdskeyval_put();
sdskeyval_get();
#endif
typedef int kv_id;
typedef enum {
KV_INT,
KV_UINT,
KV_FLOAT,
KV_DOUBLE,
KV_STRING,
KV_BULK,
} kv_type;
// define client/server key/value types here (before mercury includes)
// kv-client POD types supported:
// integer (e.g. int, long, long long)
// unsigned integer (e.g. unsigned int, long, and long long)
// floating point single precision (e.g. float)
// floating point double precision (e.g. double)
// string (use hg_string_t)
// key type
typedef uint64_t kv_key_t;
// value type for POD put/get interface (e.g. long)
typedef int kv_value_t;
// kv-client bulk data:
// bulk (pack/unpack values in/out of memory buffer)
// see datastore.h/cc for implementation (ds_bulk_t)
/* do we need one for server, one for client? */
typedef struct kv_context_s {
......@@ -44,39 +53,31 @@ typedef struct kv_context_s {
hg_handle_t bulk_get_handle; // necessary?
hg_handle_t bench_handle;
hg_handle_t shutdown_handle;
/* some keyval dodad goes here so the server can discriminate. Seems
* like it should be some universal identifier we can share with other
* clients */
/* some keyval dodad goes here so the server can discriminate
* seems like it should be some universal identifier we can
* share with other clients */
kv_id kv;
} kv_context;
/* struggling a bit with types. We'll need to create one of these for every
* type? */
/* struggling a bit with types */
MERCURY_GEN_PROC(put_in_t,
((int32_t)(key))\
((int32_t)(value)) )
MERCURY_GEN_PROC(put_out_t, ((int32_t)(ret)) )
((uint64_t)(key)) ((int32_t)(value)))
MERCURY_GEN_PROC(put_out_t, ((int32_t)(ret)))
DECLARE_MARGO_RPC_HANDLER(put_handler)
MERCURY_GEN_PROC(get_in_t,
((int32_t)(key)) )
((uint64_t)(key)))
MERCURY_GEN_PROC(get_out_t,
((int32_t)(value)) ((int32_t)(ret)) )
((int32_t)(value)) ((int32_t)(ret)))
DECLARE_MARGO_RPC_HANDLER(get_handler)
MERCURY_GEN_PROC(open_in_t,
((hg_string_t)(name)) \
((uint32_t) (keytype)) \
((uint32_t) (valtype)))
((hg_string_t)(name)))
MERCURY_GEN_PROC(open_out_t, ((int32_t)(ret)))
DECLARE_MARGO_RPC_HANDLER(open_handler)
MERCURY_GEN_PROC(close_in_t,
((int32_t)(x)) \
((int32_t)(y)) )
MERCURY_GEN_PROC(close_out_t, ((int32_t)(ret)) )
MERCURY_GEN_PROC(close_out_t, ((int32_t)(ret)))
DECLARE_MARGO_RPC_HANDLER(close_handler)
MERCURY_GEN_PROC(bench_in_t, ((int32_t)(count)) )
......@@ -88,10 +89,10 @@ typedef struct {
double overhead;
} bench_result;
static inline hg_return_t hg_proc_bench_result( hg_proc_t proc, bench_result *result)
static inline hg_return_t hg_proc_bench_result(hg_proc_t proc, bench_result *result)
{
/* TODO: needs a portable encoding */
return(hg_proc_memcpy(proc, result, sizeof(*result))) ;
/* TODO: needs a portable encoding */
return(hg_proc_memcpy(proc, result, sizeof(*result)));
}
DECLARE_MARGO_RPC_HANDLER(bench_handler)
......@@ -110,8 +111,7 @@ MERCURY_GEN_PROC(bulk_get_in_t,
((uint64_t)(size)) \
((hg_bulk_t)(bulk_handle)) )
MERCURY_GEN_PROC(bulk_get_out_t,
((uint64_t)(size)) \
((int32_t)(ret)))
((uint64_t)(size)) ((int32_t)(ret)))
DECLARE_MARGO_RPC_HANDLER(bulk_get_handler)
......@@ -120,8 +120,6 @@ kv_context * kv_server_register(margo_instance_id mid);
DECLARE_MARGO_RPC_HANDLER(shutdown_handler)
DECLARE_MARGO_RPC_HANDLER(shutdown_handler)
/* both the same: should probably move to common */
hg_return_t kv_client_deregister(kv_context *context);
hg_return_t kv_server_deregister(kv_context *context);
......@@ -131,12 +129,11 @@ hg_return_t kv_server_wait_for_shutdown(kv_context *context);
/* client-side routines wrapping up all the RPC stuff */
hg_return_t kv_client_signal_shutdown(kv_context *context);
hg_return_t kv_open(kv_context *context, const char *server, const char *db_name,
kv_type keytype, kv_type valtype);
hg_return_t kv_open(kv_context *context, const char *server, const char *db_name);
hg_return_t kv_put(kv_context *context, void *key, void *value);
hg_return_t kv_bulk_put(kv_context *context, void *key, void *data, uint64_t *data_size);
hg_return_t kv_bulk_put(kv_context *context, void *key, void *data, size_t *data_size);
hg_return_t kv_get(kv_context *context, void *key, void *value);
hg_return_t kv_bulk_get(kv_context *context, void *key, void *data, uint64_t *data_size);
hg_return_t kv_bulk_get(kv_context *context, void *key, void *data, size_t *data_size);
hg_return_t kv_close(kv_context *context);
bench_result *kv_benchmark(kv_context *context, int count);
......@@ -144,3 +141,4 @@ bench_result *kv_benchmark(kv_context *context, int count);
}
#endif
#endif // sds_keyval_h
......@@ -63,7 +63,7 @@ int main(int argc, char **argv)
size_t items = atoi(argv[1]);
context = kv_client_register(NULL);
kv_open(context, argv[2], NULL, KV_INT, KV_INT);
kv_open(context, argv[2], NULL);
RandomInsertSpeedTest(context, items, &rpc);
print_results(&rpc);
......
......@@ -2,32 +2,35 @@
#include <assert.h>
int main(int argc, char **argv) {
int ret;
kv_context * context = kv_client_register(NULL);
int ret;
kv_context * context = kv_client_register(NULL);
/* open */
ret = kv_open(context, argv[1], "booger", KV_INT, KV_INT);
/* open */
ret = kv_open(context, argv[1], "booger");
/* put */
int key = 10;
int val = 10;
ret = kv_put(context, &key, &val);
/* put */
int key = 10;
int val = 10;
ret = kv_put(context, &key, &val);
/* get */
int remote_val;
ret = kv_get(context, &key, &remote_val);
printf("key: %d in: %d out: %d\n", key, val, remote_val);
/* get */
int remote_val;
ret = kv_get(context, &key, &remote_val);
printf("key: %d in: %d out: %d\n", key, val, remote_val);
/* close */
ret = kv_close(context);
/* signal server */
ret = kv_client_signal_shutdown(context);
/* close */
ret = kv_close(context);
/* benchmark doesn't require an open keyval */
bench_result *output;
output = kv_benchmark(context, 1000);
printf("insert: %zd keys in %f seconds: %f Million-insert per sec\n",
output->nkeys, output->insert_time,
output->nkeys/(output->insert_time*1024*1024) );
free(output);
/* benchmark doesn't require an open keyval */
bench_result *output;
output = kv_benchmark(context, 1000);
printf("insert: %zd keys in %f seconds: %f Million-insert per sec\n",
output->nkeys, output->insert_time,
output->nkeys/(output->insert_time*1024*1024) );
free(output);
// kv_client_deregister(context);
kv_client_deregister(context);
}
......@@ -27,11 +27,6 @@
} \
} while(0)
static void usage()
{
fprintf(stderr, "Usage: test-mpi <addr>\n");
}
int main(int argc, char *argv[])
{
int rank;
......@@ -71,7 +66,6 @@ int main(int argc, char *argv[])
printf("rank %d: server deregistered\n", rank);
}
else {
int sleep_time = 0;
char server_addr_str[128];
char client_addr_str_in[128];
char client_addr_str_out[128];
......@@ -99,8 +93,8 @@ int main(int argc, char *argv[])
printf("client (rank %d): client addr_str: %s\n", rank, client_addr_str_out);
// open specified "DB" (pass in the server's address)
const char *db = "minima_store";
hret = kv_open(context, server_addr_str, (char*)db, KV_UINT, KV_BULK);
const char *db = "db/minima_store";
hret = kv_open(context, server_addr_str, (char*)db);
DIE_IF(hret != HG_SUCCESS, "kv_open");