Commit 60d0d4dd authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

Merge branch 'dev-benchmark' into 'master'

Dev benchmark

See merge request !8
parents fc793933 cfe8fae8
...@@ -8,6 +8,10 @@ AM_CPPFLAGS = -I${srcdir}/src -I${srcdir}/include ...@@ -8,6 +8,10 @@ AM_CPPFLAGS = -I${srcdir}/src -I${srcdir}/include
bin_PROGRAMS = bin/sdskv-server-daemon \ bin_PROGRAMS = bin/sdskv-server-daemon \
bin/sdskv-shutdown bin/sdskv-shutdown
if BUILD_BENCHMARK
bin_PROGRAMS += bin/sdskv-benchmark
endif
check_PROGRAMS = test/sdskv-open-test \ check_PROGRAMS = test/sdskv-open-test \
test/sdskv-list-db-test \ test/sdskv-list-db-test \
test/sdskv-put-test \ test/sdskv-put-test \
...@@ -34,6 +38,13 @@ bin_sdskv_shutdown_DEPENDENCIES = lib/libsdskv-client.la ...@@ -34,6 +38,13 @@ bin_sdskv_shutdown_DEPENDENCIES = lib/libsdskv-client.la
bin_sdskv_shutdown_LDFLAGS = -Llib -lsdskv-client bin_sdskv_shutdown_LDFLAGS = -Llib -lsdskv-client
bin_sdskv_shutdown_LDADD = ${LIBS} -lsdskv-client bin_sdskv_shutdown_LDADD = ${LIBS} -lsdskv-client
if BUILD_BENCHMARK
bin_sdskv_benchmark_SOURCES = src/sdskv-benchmark.cc
bin_sdskv_benchmark_DEPENDENCIES = lib/libsdskv-client.la lib/libsdskv-server.la
bin_sdskv_benchmark_LDFLAGS = -Llib -lsdskv-client -lsdskv-server
bin_sdskv_benchmark_LDADD = ${LIBS} -lsdskv-client -lsdskv-server ${SERVER_LIBS}
endif
#lib_LTLIBRARIES = lib/libkvclient.la \ #lib_LTLIBRARIES = lib/libkvclient.la \
# lib/libkvserver.la \ # lib/libkvserver.la \
# lib/libkvgroupclient.la \ # lib/libkvgroupclient.la \
......
...@@ -159,6 +159,10 @@ if test "x${bwtree_backend}" == xyes ; then ...@@ -159,6 +159,10 @@ if test "x${bwtree_backend}" == xyes ; then
CXXFLAGS="-pthread -g -Wall -mcx16 -Wno-invalid-offsetof ${CXXFLAGS}" CXXFLAGS="-pthread -g -Wall -mcx16 -Wno-invalid-offsetof ${CXXFLAGS}"
fi fi
AM_CONDITIONAL([BUILD_BDB], [test "x${berkelydb_backend}" == xyes])
AM_CONDITIONAL([BUILD_LEVELDB], [test "x${leveldb_backend}" == xyes])
AM_CONDITIONAL([BUILD_BWTREE], [test "x${bwtree_backend}" == xyes])
AC_ARG_ENABLE(remi, AC_ARG_ENABLE(remi,
[AS_HELP_STRING([--enable-remi],[Enable REMI (migration) support @<:@default=no@:>@])], [AS_HELP_STRING([--enable-remi],[Enable REMI (migration) support @<:@default=no@:>@])],
[case "${enableval}" in [case "${enableval}" in
...@@ -182,9 +186,22 @@ else ...@@ -182,9 +186,22 @@ else
fi fi
AC_SUBST(USE_REMI) AC_SUBST(USE_REMI)
AM_CONDITIONAL([BUILD_BDB], [test "x${berkelydb_backend}" == xyes]) AC_ARG_ENABLE(benchmark,
AM_CONDITIONAL([BUILD_LEVELDB], [test "x${leveldb_backend}" == xyes]) [AS_HELP_STRING([--enable-benchmark],[Build SDSKV benchmark @<:@default=no@:>@])],
AM_CONDITIONAL([BUILD_BWTREE], [test "x${bwtree_backend}" == xyes]) [case "${enableval}" in
yes) enable_benchmark="yes" ;;
no) enable_benchmark="no" ;;
*) AC_MSG_ERROR(bad value ${enableval} for --enable-benchmark) ;;
esac],
[enable_benchmark="no"]
)
AM_CONDITIONAL(BUILD_BENCHMARK, test x$enable_benchmark = xyes)
if test "$enable_benchmark" = "yes"; then
PKG_CHECK_MODULES(JSONCPP, jsoncpp)
LIBS="$JSONCPP_LIBS $LIBS"
CPPFLAGS="$JSONCPP_CFLAGS $CPPFLAGS"
CFLAGS="$JSONCPP_CFLAGS $CFLAGS"
fi
SERVER_LIBS="$SERVER_LIBS_PKG $SERVER_LIBS_EXT -lstdc++" SERVER_LIBS="$SERVER_LIBS_PKG $SERVER_LIBS_EXT -lstdc++"
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
#include <sdskv-common.hpp> #include <sdskv-common.hpp>
#define _CHECK_RET(__ret) \ #define _CHECK_RET(__ret) \
if(__ret != BAKE_SUCCESS) throw exception(__ret) if(__ret != SDSKV_SUCCESS) throw exception(__ret)
namespace sdskv { namespace sdskv {
...@@ -15,7 +15,7 @@ namespace sdskv { ...@@ -15,7 +15,7 @@ namespace sdskv {
class provider { class provider {
margo_instance_id m_mid = MARGO_INSTANCE_NULL; margo_instance_id m_mid = MARGO_INSTANCE_NULL;
sdskv_provider_t m_provider = SDSKV_PROVIDER_NULL; sdskv_provider_t m_provider = NULL;
/** /**
* @brief Constructor is private. Use the create() factory method. * @brief Constructor is private. Use the create() factory method.
...@@ -55,7 +55,8 @@ class provider { ...@@ -55,7 +55,8 @@ class provider {
ABT_pool pool = SDSKV_ABT_POOL_DEFAULT) ABT_pool pool = SDSKV_ABT_POOL_DEFAULT)
{ {
auto p = new provider(mid, provider_id, pool); auto p = new provider(mid, provider_id, pool);
margo_provider_push_finalize_callback(mid, this, &finalize_callback, this); margo_provider_push_finalize_callback(mid, p, &finalize_callback, p);
return p;
} }
/** /**
...@@ -82,7 +83,7 @@ class provider { ...@@ -82,7 +83,7 @@ class provider {
* @brief Destructor. Deregisters the provider. * @brief Destructor. Deregisters the provider.
*/ */
~provider() { ~provider() {
margo_provider_pop_finalize_callback(mid, this); margo_provider_pop_finalize_callback(m_mid, this);
sdskv_provider_destroy(m_provider); sdskv_provider_destroy(m_provider);
} }
...@@ -191,5 +192,5 @@ class provider { ...@@ -191,5 +192,5 @@ class provider {
}; };
} }
#undef _CHECK_RET
#endif #endif
{
"protocol" : "tcp",
"seed" : 0,
"server" : {
"use-progress-thread" : false,
"rpc-thread-count" : 0,
"database" : {
"type" : "map",
"name" : "benchmark-db",
"path" : "/dev/shm"
}
},
"benchmarks" : [
{
"type" : "put",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : [ 8, 32 ],
"val-sizes" : [ 24, 48 ],
"erase-on-teardown" : true
},
{
"type" : "put-multi",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : [ 8, 32 ],
"val-sizes" : [ 24, 48 ],
"erase-on-teardown" : true
},
{
"type" : "get",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : 64,
"val-sizes" : 128,
"erase-on-teardown" : true
},
{
"type" : "get-multi",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : 32,
"val-sizes" : [ 56, 64 ],
"erase-on-teardown" : true
},
{
"type" : "length",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : 64,
"val-sizes" : 128,
"erase-on-teardown" : true
},
{
"type" : "length-multi",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : 32,
"val-sizes" : [ 56, 64 ],
"erase-on-teardown" : true
},
{
"type" : "erase",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : 64,
"val-sizes" : 128
},
{
"type" : "erase-multi",
"repetitions" : 10,
"num-entries" : 30,
"key-sizes" : 32,
"val-sizes" : [ 56, 64 ]
}
]
}
#include <iostream>
#include <iomanip>
#include <cmath>
#include <algorithm>
#include <numeric>
#include <fstream>
#include <map>
#include <functional>
#include <memory>
#include <mpi.h>
#include <json/json.h>
#include <sdskv-client.hpp>
#include <sdskv-server.hpp>
using RemoteDatabase = sdskv::database;
/**
* Helper function to generate random strings of a certain length.
* These strings are readable.
*/
static std::string gen_random_string(size_t len) {
static const char alphanum[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
std::string s(len, ' ');
for (unsigned i = 0; i < len; ++i) {
s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];
}
return s;
}
template<typename T>
class BenchmarkRegistration;
/**
* The AbstractBenchmark class describes an interface that a benchmark object
* needs to satisfy. This interface has a setup, execute, and teardown
* methods. AbstractBenchmark also acts as a factory to create concrete instances.
*/
class AbstractBenchmark {
MPI_Comm m_comm; // communicator gathering all clients
RemoteDatabase& m_remote_db; // remote database
template<typename T>
friend class BenchmarkRegistration;
using benchmark_factory_function = std::function<
std::unique_ptr<AbstractBenchmark>(Json::Value&, MPI_Comm, RemoteDatabase&)>;
static std::map<std::string, benchmark_factory_function> s_benchmark_factories;
protected:
RemoteDatabase& remoteDatabase() { return m_remote_db; }
const RemoteDatabase& remoteDatabase() const { return m_remote_db; }
MPI_Comm comm() const { return m_comm; }
public:
AbstractBenchmark(MPI_Comm c, RemoteDatabase& rdb)
: m_comm(c)
, m_remote_db(rdb) {}
virtual ~AbstractBenchmark() = default;
virtual void setup() = 0;
virtual void execute() = 0;
virtual void teardown() = 0;
/**
* @brief Factory function used to create benchmark instances.
*/
template<typename ... T>
static std::unique_ptr<AbstractBenchmark> create(const std::string& type, T&& ... args) {
auto it = s_benchmark_factories.find(type);
if(it == s_benchmark_factories.end())
throw std::invalid_argument(type+" benchmark type unknown");
return (it->second)(std::forward<T>(args)...);
}
};
/**
* @brief The mechanism bellow is used to provide the REGISTER_BENCHMARK macro,
* which registers a child class of AbstractBenchmark and allows AbstractBenchmark::create("type", ...)
* to return an instance of this concrete child class.
*/
template<typename T>
class BenchmarkRegistration {
public:
BenchmarkRegistration(const std::string& type) {
AbstractBenchmark::s_benchmark_factories[type] =
[](Json::Value& config, MPI_Comm comm, RemoteDatabase& rdb) {
return std::make_unique<T>(config, comm, rdb);
};
}
};
std::map<std::string, AbstractBenchmark::benchmark_factory_function> AbstractBenchmark::s_benchmark_factories;
#define REGISTER_BENCHMARK(__name, __class) \
static BenchmarkRegistration<__class> __class##_registration(__name)
class AbstractAccessBenchmark : public AbstractBenchmark {
protected:
uint64_t m_num_entries = 0;
std::pair<size_t, size_t> m_key_size_range;
std::pair<size_t, size_t> m_val_size_range;
bool m_erase_on_teardown;
public:
template<typename ... T>
AbstractAccessBenchmark(Json::Value& config, T&& ... args)
: AbstractBenchmark(std::forward<T>(args)...) {
// read the configuration
m_num_entries = config["num-entries"].asUInt64();
if(config["key-sizes"].isIntegral()) {
auto x = config["key-sizes"].asUInt64();
m_key_size_range = { x, x+1 };
} else if(config["key-sizes"].isArray() && config["key-sizes"].size() == 2) {
auto x = config["key-sizes"][0].asUInt64();
auto y = config["key-sizes"][1].asUInt64();
if(x > y) throw std::range_error("invalid key-sizes range");
m_key_size_range = { x, y };
} else {
throw std::range_error("invalid key-sizes range or value");
}
if(config["val-sizes"].isIntegral()) {
auto x = config["val-sizes"].asUInt64();
m_val_size_range = { x, x+1 };
} else if(config["val-sizes"].isArray() && config["val-sizes"].size() == 2) {
auto x = config["val-sizes"][0].asUInt64();
auto y = config["val-sizes"][1].asUInt64();
if(x >= y) throw std::range_error("invalid val-sizes range");
m_val_size_range = { x, y };
} else {
throw std::range_error("invalid val-sizes range or value");
}
m_erase_on_teardown = config["erase-on-teardown"].asBool();
}
};
/**
* PutBenchmark executes a series of PUT operations and measures their duration.
*/
class PutBenchmark : public AbstractAccessBenchmark {
protected:
std::vector<std::string> m_keys;
std::vector<std::string> m_vals;
public:
template<typename ... T>
PutBenchmark(T&& ... args)
: AbstractAccessBenchmark(std::forward<T>(args)...) { }
virtual void setup() override {
// generate key/value pairs and store them in the local
m_keys.reserve(m_num_entries);
m_vals.reserve(m_num_entries);
for(unsigned i=0; i < m_num_entries; i++) {
size_t ksize = m_key_size_range.first + (rand() % (m_key_size_range.second - m_key_size_range.first));
m_keys.push_back(gen_random_string(ksize));
size_t vsize = m_val_size_range.first + (rand() % (m_val_size_range.second - m_val_size_range.first));
m_vals.push_back(gen_random_string(vsize));
}
}
virtual void execute() override {
// execute PUT operations
auto& db = remoteDatabase();
for(unsigned i=0; i < m_num_entries; i++) {
auto& key = m_keys[i];
auto& val = m_vals[i];
db.put(key, val);
}
}
virtual void teardown() override {
if(m_erase_on_teardown) {
// erase all the keys from the database
auto& db = remoteDatabase();
for(unsigned i=0; i < m_num_entries; i++) {
db.erase(m_keys[i]);
}
}
// erase keys and values from the local vectors
m_keys.resize(0);
m_vals.resize(0);
}
};
REGISTER_BENCHMARK("put", PutBenchmark);
/**
* PutMultiBenchmark inherites from PutBenchmark and does the same but
* executes a PUT-MULTI instead of a PUT.
*/
class PutMultiBenchmark : public PutBenchmark {
public:
template<typename ... T>
PutMultiBenchmark(T&& ... args)
: PutBenchmark(std::forward<T>(args)...) {}
virtual void execute() override {
auto& db = remoteDatabase();
db.put(m_keys, m_vals);
}
};
REGISTER_BENCHMARK("put-multi", PutMultiBenchmark);
/**
* GetBenchmark executes a series of GET operations and measures their duration.
*/
class GetBenchmark : public AbstractAccessBenchmark {
protected:
std::vector<std::string> m_keys;
std::vector<std::string> m_vals;
public:
template<typename ... T>
GetBenchmark(T&& ... args)
: AbstractAccessBenchmark(std::forward<T>(args)...) {}
virtual void setup() override {
// generate key/value pairs and store them in the local
m_keys.reserve(m_num_entries);
m_vals.reserve(m_num_entries);
for(unsigned i=0; i < m_num_entries; i++) {
size_t ksize = m_key_size_range.first + (rand() % (m_key_size_range.second - m_key_size_range.first));
m_keys.push_back(gen_random_string(ksize));
size_t vsize = m_val_size_range.first + (rand() % (m_val_size_range.second - m_val_size_range.first));
m_vals.push_back(gen_random_string(vsize));
}
// execute PUT operations (not part of the measure)
auto& db = remoteDatabase();
for(unsigned i=0; i < m_num_entries; i++) {
auto& key = m_keys[i];
auto& val = m_vals[i];
db.put(key, val);
}
}
virtual void execute() override {
// execute GET operations
auto& db = remoteDatabase();
std::string val(m_val_size_range.second-1, 0);
for(unsigned i=0; i < m_num_entries; i++) {
auto& key = m_keys[i];
db.get(key, val);
val.resize(m_val_size_range.second-1, 0);
}
}
virtual void teardown() override {
if(m_erase_on_teardown) {
// erase all the keys from the database
auto& db = remoteDatabase();
for(unsigned i=0; i < m_num_entries; i++) {
db.erase(m_keys[i]);
}
}
// erase keys and values from the local vectors
m_keys.resize(0);
m_vals.resize(0);
}
};
REGISTER_BENCHMARK("get", GetBenchmark);
/**
* GetMultiBenchmark inherites from GetBenchmark and does the same but
* executes a GET-MULTI instead of a GET.
*/
class GetMultiBenchmark : public GetBenchmark {
public:
template<typename ... T>
GetMultiBenchmark(T&& ... args)
: GetBenchmark(std::forward<T>(args)...) {}
virtual void execute() override {
auto& db = remoteDatabase();
std::vector<std::string> vals(m_num_entries, std::string(m_val_size_range.second-1, 0));
db.get(m_keys, vals);
}
};
REGISTER_BENCHMARK("get-multi", GetMultiBenchmark);
/**
* LengthBenchmark executes a series of LENGTH operations and measures their duration.
*/
class LengthBenchmark : public AbstractAccessBenchmark {
protected:
std::vector<std::string> m_keys;
std::vector<std::string> m_vals;
public:
template<typename ... T>
LengthBenchmark(T&& ... args)
: AbstractAccessBenchmark(std::forward<T>(args)...) {}
virtual void setup() override {
// generate key/value pairs and store them in the local
m_keys.reserve(m_num_entries);
m_vals.reserve(m_num_entries);
for(unsigned i=0; i < m_num_entries; i++) {
size_t ksize = m_key_size_range.first + (rand() % (m_key_size_range.second - m_key_size_range.first));
m_keys.push_back(gen_random_string(ksize));
size_t vsize = m_val_size_range.first + (rand() % (m_val_size_range.second - m_val_size_range.first));
m_vals.push_back(gen_random_string(vsize));
}
// execute PUT operations (not part of the measure)
auto& db = remoteDatabase();
for(unsigned i=0; i < m_num_entries; i++) {
auto& key = m_keys[i];
auto& val = m_vals[i];
db.put(key, val);
}
}
virtual void execute() override {
// execute LENGTH operations
auto& db = remoteDatabase();
for(unsigned i=0; i < m_num_entries; i++) {
auto& key = m_keys[i];
db.length(key);
}
}
virtual void teardown() override {
if(m_erase_on_teardown) {
// erase all the keys from the database
auto& db = remoteDatabase();
for(unsigned i=0; i < m_num_entries; i++) {
db.erase(m_keys[i]);
}
}
// erase keys and values from the local vectors
m_keys.resize(0);
m_vals.resize(0);
}
};
REGISTER_BENCHMARK("length", LengthBenchmark);
/**
* LengthMultiBenchmark inherites from LengthBenchmark and does the same but
* executes a LENGTH-MULTI instead of a LENGTH.
*/
class LengthMultiBenchmark : public LengthBenchmark {
public:
template<typename ... T>
LengthMultiBenchmark(T&& ... args)
: LengthBenchmark(std::forward<T>(args)...) {}
virtual void execute() override {
auto& db = remoteDatabase();
std::vector<hg_size_t> sizes(m_num_entries);
db.length(m_keys, sizes);
}
};
REGISTER_BENCHMARK("length-multi", LengthMultiBenchmark);
/**
* EraseBenchmark executes a series of ERASE operations and measures their duration.
*/
class EraseBenchmark : public AbstractBenchmark {
protected:
uint64_t m_num_entries = 0;
std::pair<size_t, size_t> m_key_size_range;
std::pair<size_t, size_t> m_val_size_range;
std::vector<std::string> m_keys;
std::vector<std::string> m_vals;
public:
template<typename ... T>
EraseBenchmark(Json::Value& config, T&& ... args)
: AbstractBenchmark(std::forward<T>(args)...) {
// read the configuration
m_num_entries = config["num-entries"].asUInt64();
if(config["key-sizes"].isIntegral()) {
auto x = config["key-sizes"].asUInt64();
m_key_size_range = { x, x+1 };
} else if(config["key-sizes"].isArray() && config["key-sizes"].size() == 2) {
auto x = config["key-sizes"][0].asUInt64();
auto y = config["key-sizes"][1].asUInt64();