Commit 0885eaad authored by Matthieu Dorier's avatar Matthieu Dorier

Merge branch 'dev-put-multi-packed' into 'master'

Added "packed" versions of "put_multi", "get_multi" and "length_multi"

See merge request !10
parents 0eccaf2c 05468e39
......@@ -24,6 +24,7 @@ check_PROGRAMS = test/sdskv-open-test \
test/sdskv-custom-cmp-test \
test/sdskv-migrate-test \
test/sdskv-multi-test \
test/sdskv-packed-test \
test/sdskv-cxx-test \
test/sdskv-custom-server-daemon
......@@ -127,6 +128,7 @@ TESTS = test/basic.sh \
test/migrate-test.sh \
test/custom-cmp-test.sh \
test/multi-test.sh \
test/packed-test.sh \
test/cxx-test.sh
TESTS_ENVIRONMENT = TIMEOUT="$(TIMEOUT)" \
......@@ -185,6 +187,10 @@ test_sdskv_multi_test_SOURCES = test/sdskv-multi-test.cc
test_sdskv_multi_test_DEPENDENCIES = lib/libsdskv-client.la
test_sdskv_multi_test_LDFLAGS = -Llib -lsdskv-client
test_sdskv_packed_test_SOURCES = test/sdskv-packed-test.cc
test_sdskv_packed_test_DEPENDENCIES = lib/libsdskv-client.la
test_sdskv_packed_test_LDFLAGS = -Llib -lsdskv-client
test_sdskv_cxx_test_SOURCES = test/sdskv-cxx-test.cc
test_sdskv_cxx_test_DEPENDENCIES = lib/libsdskv-client.la
test_sdskv_cxx_test_LDFLAGS = -Llib -lsdskv-client
......
......@@ -178,6 +178,28 @@ int sdskv_put_multi(sdskv_provider_handle_t provider,
size_t num, const void* const* keys, const hg_size_t* ksizes,
const void* const* values, const hg_size_t *vsizes);
/**
* @brief Puts multiple key/value pairs into the database.
* This method will send all the key/value pairs in batch,
* thus optimizing transfers by avoiding many RPC round trips.
* This version of put_multi assumes that the keys are packed
* in back to back in a single buffer, and so are the values.
*
* @param provider provider handle managing the database
* @param db_id targeted database id
* @param num number of key/value pairs to put
* @param packed_keys buffer containing the keys
* @param ksizes array of key sizes
* @param packed_values buffer containing the values
* @param vsizes array of value sizes
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
int sdskv_put_packed(sdskv_provider_handle_t provider,
sdskv_database_id_t db_id,
size_t num, const void* packed_keys, const hg_size_t *ksizes,
const void* packed_values, const hg_size_t *vsizes);
/**
* @brief Gets the value associated with a given key.
* vsize needs to be set to the current size of the allocated
......@@ -236,6 +258,25 @@ int sdskv_get_multi(sdskv_provider_handle_t provider,
size_t num, const void* const* keys, const hg_size_t* ksizes,
void** values, hg_size_t *vsizes);
/**
* @brief Get multiple values into a single packed buffer.
*
* @param[in] provider provider handle
* @param[in] db_id database id
* @param[inout] num number of values to retrieve, number of values actually retrieved
* @param[in] keys buffer of packed keys to retrieve
* @param[in] ksizes size of the keys
* @param[in] vbufsize size of the buffer allocated for the values
* @param[out] values buffer allocated to receive packed values
* @param[out] vsizes sizes of the values
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
int sdskv_get_packed(sdskv_provider_handle_t provider,
sdskv_database_id_t db_id,
size_t* num, const void* packed_keys, const hg_size_t* ksizes,
hg_size_t vbufsize, void* packed_values, hg_size_t *vsizes);
/**
* @brief Gets the length of a value associated with a given key.
*
......@@ -271,6 +312,26 @@ int sdskv_length_multi(sdskv_provider_handle_t handle,
const void* const* keys, const hg_size_t* ksizes,
hg_size_t *vsizes);
/**
* @brief Gets the length of values associated with multiple keys.
* If a particular key does not exists, this function will set the length
* of its value to 0 (so the user needs another way to differenciate
* between a key that does not exists and a 0-sized value).
*
* @param[in] handle provider handle
* @param[in] db_id database id
* @param[in] num number of keys
* @param[in] keys buffer of packed keys
* @param[in] ksizes array of key sizes
* @param[out] vsizes array where to put value sizes
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
int sdskv_length_packed(sdskv_provider_handle_t handle,
sdskv_database_id_t db_id, size_t num,
const void* packed_keys, const hg_size_t* ksizes,
hg_size_t *vsizes);
/**
* @brief Checks if the given key exists in the database.
*
......
......@@ -324,6 +324,39 @@ class client {
put_multi(db, kdata, ksizes, vdata, vsizes);
}
//////////////////////////
// PUT_PACKED methods
//////////////////////////
/**
* @brief Equivalent to sdskv_put_packed.
*
* @param db Database instance.
* @param count Number of key/val pairs.
* @param keys Buffer of keys.
* @param ksizes Array of key sizes.
* @param values Buffer of values.
* @param vsizes Array of value sizes.
*/
void put_packed(const database& db,
hg_size_t count, const void* keys, const hg_size_t* ksizes,
const void* values, const hg_size_t *vsizes) const;
/**
* @brief Version of put taking std::strings instead of pointers.
*
* @param db Database instance.
* @param keys Vector of pointers to keys.
* @param ksizes Vector of key sizes.
* @param values Vector of pointers to values.
* @param vsizes Vector of value sizes.
*/
inline void put_packed(const database& db,
const std::string& packed_keys, const std::vector<hg_size_t>& ksizes,
const std::string& packed_values, const std::vector<hg_size_t>& vsizes) const {
put_packed(db, ksizes.size(), packed_keys.data(), ksizes.data(), packed_values.data(), vsizes.data());
}
//////////////////////////
// EXISTS methods
//////////////////////////
......@@ -445,6 +478,38 @@ class client {
return vsizes;
}
//////////////////////////
// LENGTH_PACKED methods
//////////////////////////
/**
* @brief Equivalent to sdskv_length_packed.
*
* @param db Database instance.
* @param num Number of keys.
* @param keys Packed keys.
* @param ksizes Array of key sizes.
* @param vsizes Resulting value sizes.
*/
bool length_packed(const database& db,
hg_size_t num, const void* keys,
const hg_size_t* ksizes, hg_size_t* vsizes) const;
/**
* Version of length_packed that takes an std::string as packed buffer.
*
* @param db Database instance.
* @param keys Packed keys.
* @param vsizes Resulting vector of value sizes.
*/
inline bool length_multi(const database& db,
const std::string& keys,
const std::vector<hg_size_t>& ksizes,
std::vector<hg_size_t>& vsizes) const {
vsizes.resize(ksizes.size());
return length_packed(db, ksizes.size(), keys.data(), ksizes.data(), vsizes.data());
}
//////////////////////////
// GET methods
//////////////////////////
......@@ -652,6 +717,49 @@ class client {
return values;
}
//////////////////////////
// GET_PACKED methods
//////////////////////////
/**
* @brief Equivalent to sdskv_get_packed.
*
* @param db Database instance.
* @param count Number of key/val pairs.
* @param keys Buffer of packed keys.
* @param ksizes Array of key sizes.
* @param valbufsize Size of the value buffer.
* @param values Buffer of packed values.
* @param vsizes Array of sizes of value buffers.
*/
bool get_packed(const database& db,
hg_size_t* count, const void* keys, const hg_size_t* ksizes,
hg_size_t valbufsize, void* values, hg_size_t *vsizes) const;
/**
* @brief Get multiple key/val pairs using std::string packed buffers
* and std::vector<hg_size_t> of sizes. The value buffer must be
* pre-allocated. vsizes will be resized to the right number of retrieved
* values.
*
* @param db Database instance.
* @param keys Vector of key addresses.
* @param ksizes Vector of key sizes.
* @param values Vector of value addresses.
* @param vsizes Vector of value sizes.
*/
inline bool get_packed(const database& db,
const std::string& packed_keys, const std::vector<hg_size_t>& ksizes,
std::string& packed_values, std::vector<hg_size_t>& vsizes) const {
hg_size_t count = ksizes.size();
vsizes.resize(count);
bool b = get_packed(db, &count, packed_keys.data(), ksizes.data(),
packed_values.size(), const_cast<char*>(packed_values.data()), vsizes.data());
vsizes.resize(count);
return b;
}
//////////////////////////
// ERASE methods
//////////////////////////
......@@ -1305,6 +1413,14 @@ class database {
m_ph.m_client->put_multi(*this, std::forward<T>(args)...);
}
/**
* @brief @see client::put_packed.
*/
template<typename ... T>
void put_packed(T&& ... args) const {
m_ph.m_client->put_packed(*this, std::forward<T>(args)...);
}
/**
* @brief @see client::length.
*/
......@@ -1321,6 +1437,14 @@ class database {
return m_ph.m_client->length_multi(*this, std::forward<T>(args)...);
}
/**
* @brief @see client::length_packed.
*/
template<typename ... T>
decltype(auto) length_packed(T&& ... args) const {
return m_ph.m_client->length_packed(*this, std::forward<T>(args)...);
}
/**
* @brief @see client::get.
*/
......@@ -1336,6 +1460,15 @@ class database {
decltype(auto) get_multi(T&& ... args) const {
return m_ph.m_client->get_multi(*this, std::forward<T>(args)...);
}
/**
* @brief @see client::get_packed.
*/
template<typename ... T>
decltype(auto) get_packed(T&& ... args) const {
return m_ph.m_client->get_packed(*this, std::forward<T>(args)...);
}
/**
* @brief @see client::exists
......@@ -1438,6 +1571,14 @@ inline void client::put_multi(const database& db,
_CHECK_RET(ret);
}
inline void client::put_packed(const database& db,
hg_size_t count, const void* keys, const hg_size_t* ksizes,
const void* values, const hg_size_t *vsizes) const {
int ret = sdskv_put_packed(db.m_ph.m_ph, db.m_db_id,
count, keys, ksizes, values, vsizes);
_CHECK_RET(ret);
}
inline hg_size_t client::length(const database& db,
const void* key, hg_size_t ksize) const {
hg_size_t vsize;
......@@ -1463,6 +1604,15 @@ inline bool client::length_multi(const database& db,
return true;
}
inline bool client::length_packed(const database& db,
hg_size_t num, const void* keys,
const hg_size_t* ksizes, hg_size_t* vsizes) const {
int ret = sdskv_length_packed(db.m_ph.m_ph, db.m_db_id,
num, keys, ksizes, vsizes);
_CHECK_RET(ret);
return true;
}
inline bool client::get(const database& db,
const void* key, hg_size_t ksize,
void* value, hg_size_t* vsize) const {
......@@ -1481,6 +1631,15 @@ inline bool client::get_multi(const database& db,
return true;
}
inline bool client::get_packed(const database& db,
hg_size_t* count, const void* keys, const hg_size_t* ksizes,
hg_size_t valbufsize, void* values, hg_size_t *vsizes) const {
int ret = sdskv_get_packed(db.m_ph.m_ph, db.m_db_id,
count, keys, ksizes, valbufsize, values, vsizes);
_CHECK_RET(ret);
return true;
}
inline void client::erase(const database& db,
const void* key,
hg_size_t ksize) const {
......
spack:
specs:
- jsoncpp
- mpich
- mochi-margo
- mochi-abt-io
- berkeley-db
- leveldb
concretization: together
......@@ -41,6 +41,23 @@ class AbstractDataStore {
}
return ret;
}
virtual int put_packed(hg_size_t num_items,
const char* keys,
const hg_size_t* ksizes,
const char* values,
const hg_size_t* vsizes)
{
int ret = 0;
size_t keys_offset = 0;
size_t vals_offset = 0;
for(hg_size_t i=0; i < num_items; i++) {
int r = put(keys+keys_offset, ksizes[i], values+vals_offset, vsizes[i]);
ret = ret == 0 ? r : 0;
keys_offset += ksizes[i];
vals_offset += vsizes[i];
}
return ret;
}
virtual bool get(const ds_bulk_t &key, ds_bulk_t &data)=0;
virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data)=0;
virtual bool exists(const void* key, hg_size_t ksize) const = 0;
......
This diff is collapsed.
......@@ -191,6 +191,14 @@ MERCURY_GEN_PROC(put_multi_in_t, \
((hg_size_t)(vals_bulk_size)))
MERCURY_GEN_PROC(put_multi_out_t, ((int32_t)(ret)))
// ------------- PUT PACKED ------------- //
MERCURY_GEN_PROC(put_packed_in_t, \
((uint64_t)(db_id))\
((hg_size_t)(num_keys))\
((hg_size_t)(bulk_size))\
((hg_bulk_t)(bulk_handle)))
MERCURY_GEN_PROC(put_packed_out_t, ((int32_t)(ret)))
// ------------- GET MULTI ------------- //
MERCURY_GEN_PROC(get_multi_in_t, \
((uint64_t)(db_id))\
......@@ -201,6 +209,18 @@ MERCURY_GEN_PROC(get_multi_in_t, \
((hg_size_t)(vals_bulk_size)))
MERCURY_GEN_PROC(get_multi_out_t, ((int32_t)(ret)))
// ------------- GET PACKED ------------- //
MERCURY_GEN_PROC(get_packed_in_t, \
((uint64_t)(db_id))\
((hg_size_t)(num_keys))\
((hg_size_t)(keys_bulk_size))\
((hg_bulk_t)(keys_bulk_handle))\
((hg_size_t)(vals_bulk_size))\
((hg_bulk_t)(vals_bulk_handle)))
MERCURY_GEN_PROC(get_packed_out_t, \
((int32_t)(ret))\
((hg_size_t)(num_keys)))
// ------------- LENGTH MULTI ------------- //
MERCURY_GEN_PROC(length_multi_in_t, \
((uint64_t)(db_id))\
......@@ -210,7 +230,16 @@ MERCURY_GEN_PROC(length_multi_in_t, \
((hg_bulk_t)(vals_size_bulk_handle)))
MERCURY_GEN_PROC(length_multi_out_t, ((int32_t)(ret)))
// ------------- LENGTH MULTI ------------- //
// ------------- LENGTH PACKED ------------- //
MERCURY_GEN_PROC(length_packed_in_t, \
((uint64_t)(db_id))\
((hg_size_t)(num_keys))\
((hg_size_t)(in_bulk_size))\
((hg_bulk_t)(in_bulk_handle))\
((hg_bulk_t)(out_bulk_handle)))
MERCURY_GEN_PROC(length_packed_out_t, ((int32_t)(ret)))
// ------------- ERASE MULTI ------------- //
MERCURY_GEN_PROC(erase_multi_in_t, \
((uint64_t)(db_id))\
((hg_size_t)(num_keys))\
......
This diff is collapsed.
#!/bin/bash -x
if [ -z $srcdir ]; then
echo srcdir variable not set.
exit 1
fi
source $srcdir/test/test-util.sh
find_db_name
# start a server with 2 second wait,
# 20s timeout, and my_test_db as database
test_start_server 2 20 $test_db_full
sleep 1
#####################
run_to 20 test/sdskv-packed-test $svr_addr 1 $test_db_name 100
if [ $? -ne 0 ]; then
wait
exit 1
fi
wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
exit 0
/*
* (C) 2015 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <assert.h>
#include <unistd.h>
#include <margo.h>
#include <iostream>
#include <string>
#include <vector>
#include <algorithm>
#include <map>
#include "sdskv-client.h"
static std::string gen_random_string(size_t len);
int main(int argc, char *argv[])
{
char cli_addr_prefix[64] = {0};
char *sdskv_svr_addr_str;
char *db_name;
margo_instance_id mid;
hg_addr_t svr_addr;
uint8_t mplex_id;
uint32_t num_keys;
sdskv_client_t kvcl;
sdskv_provider_handle_t kvph;
hg_return_t hret;
int ret;
if(argc != 5)
{
fprintf(stderr, "Usage: %s <sdskv_server_addr> <mplex_id> <db_name> <num_keys>\n", argv[0]);
fprintf(stderr, " Example: %s tcp://localhost:1234 1 foo 1000\n", argv[0]);
return(-1);
}
sdskv_svr_addr_str = argv[1];
mplex_id = atoi(argv[2]);
db_name = argv[3];
num_keys = atoi(argv[4]);
/* initialize Margo using the transport portion of the server
* address (i.e., the part before the first : character if present)
*/
for(unsigned i=0; (i<63 && sdskv_svr_addr_str[i] != '\0' && sdskv_svr_addr_str[i] != ':'); i++)
cli_addr_prefix[i] = sdskv_svr_addr_str[i];
/* start margo */
mid = margo_init(cli_addr_prefix, MARGO_SERVER_MODE, 0, 0);
if(mid == MARGO_INSTANCE_NULL)
{
fprintf(stderr, "Error: margo_init()\n");
return(-1);
}
ret = sdskv_client_init(mid, &kvcl);
if(ret != 0)
{
fprintf(stderr, "Error: sdskv_client_init()\n");
margo_finalize(mid);
return -1;
}
/* look up the SDSKV server address */
hret = margo_addr_lookup(mid, sdskv_svr_addr_str, &svr_addr);
if(hret != HG_SUCCESS)
{
fprintf(stderr, "Error: margo_addr_lookup()\n");
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(-1);
}
/* create a SDSKV provider handle */
ret = sdskv_provider_handle_create(kvcl, svr_addr, mplex_id, &kvph);
if(ret != 0)
{
fprintf(stderr, "Error: sdskv_provider_handle_create()\n");
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(-1);
}
/* open the database */
sdskv_database_id_t db_id;
ret = sdskv_open(kvph, db_name, &db_id);
if(ret == 0) {
printf("Successfuly open database %s, id is %ld\n", db_name, db_id);
} else {
fprintf(stderr, "Error: could not open database %s\n", db_name);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(-1);
}
/* **** generate packed key/vals ***** */
std::string packed_keys;
std::vector<hg_size_t> packed_key_sizes;
std::string packed_vals;
std::vector<hg_size_t> packed_val_sizes;
std::map<std::string, std::string> reference;
size_t max_value_size = 24;
for(unsigned i=0; i < num_keys; i++) {
auto k = gen_random_string(16);
auto v = gen_random_string(3+i*(max_value_size-3)/num_keys);
reference[k] = v;
packed_keys += k;
packed_vals += v;
packed_key_sizes.push_back(k.size());
packed_val_sizes.push_back(v.size());
}
/* **** issue a put_packed ***** */
ret = sdskv_put_packed(kvph, db_id, num_keys, packed_keys.data(), packed_key_sizes.data(),
packed_vals.data(), packed_val_sizes.data());
if(ret != 0) {
fprintf(stderr, "Error: sdskv_put_packed() failed\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
printf("Successfuly inserted %d keys\n", num_keys);
/* retrieve the length of the values */
std::vector<hg_size_t> rval_len(num_keys);
ret = sdskv_length_packed(kvph, db_id, num_keys,
packed_keys.data(), packed_key_sizes.data(),
rval_len.data());
if(ret != 0) {
fprintf(stderr, "Error: sdskv_length_packed() failed\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
/* check if the lengths are correct */
for(unsigned i=0; i < num_keys; i++) {
if(rval_len[i] != packed_val_sizes[i]) {
fprintf(stderr, "Error: value %d doesn't have the right length (%ld != %ld)\n", i,
rval_len[i], packed_val_sizes[i]);
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
}
/* **** get keys **** */
/* figure out the total size needed */
unsigned total_read_size = 0;
for(auto& l : rval_len) total_read_size += l;
std::vector<char> read_values(total_read_size);
std::vector<hg_size_t> read_value_sizes(num_keys);
hg_size_t num_keys_read = num_keys;
ret = sdskv_get_packed(kvph, db_id, &num_keys_read,
packed_keys.data(), packed_key_sizes.data(),
total_read_size, read_values.data(), read_value_sizes.data());
if(ret != 0) {
fprintf(stderr, "Error: sdskv_get_packed() failed\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
/* check the keys we received against reference */
size_t koffset = 0, voffset = 0;
for(unsigned i=0; i < num_keys; i++) {
std::string vstring(read_values.data() + voffset, read_value_sizes[i]);
std::string k(packed_keys.data() + koffset, packed_key_sizes[i]);
std::cout << "Got " << k << " ===> " << vstring << "\t(size = " << vstring.size()
<< ") expected: " << reference[k] << " (size = " << reference[k].size() << ")"
<< std::endl;
if(vstring != reference[k]) {
fprintf(stderr, "Error: sdskv_get_packed() returned a value different from the reference\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
koffset += packed_key_sizes[i];
voffset += read_value_sizes[i];
}
/* shutdown the server */
ret = sdskv_shutdown_service(kvcl, svr_addr);
/**** cleanup ****/
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(ret);
}
static std::string gen_random_string(size_t len) {
static const char alphanum[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
std::string s(len, ' ');
for (unsigned i = 0; i < len; ++i) {
s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];
}
return s;
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment