Commit 6db51622 authored by Matthieu Dorier's avatar Matthieu Dorier

added multi- operations

parent a506507d
......@@ -18,6 +18,7 @@ check_PROGRAMS = test/sdskv-open-test \
test/sdskv-list-keys-prefix-test \
test/sdskv-custom-cmp-test \
test/sdskv-migrate-test \
test/sdskv-multi-test \
test/sdskv-custom-server-daemon
bin_sdskv_server_daemon_SOURCES = src/sdskv-server-daemon.c
......@@ -107,7 +108,8 @@ TESTS = test/basic.sh \
test/list-keyvals-test.sh \
test/list-keys-prefix-test.sh \
test/migrate-test.sh \
test/custom-cmp-test.sh
test/custom-cmp-test.sh \
test/multi-test.sh
TESTS_ENVIRONMENT = TIMEOUT="$(TIMEOUT)" \
MKTEMP="$(MKTEMP)"
......@@ -157,6 +159,10 @@ test_sdskv_migrate_test_SOURCES = test/sdskv-migrate-test.cc
test_sdskv_migrate_test_DEPENDENCIES = lib/libsdskv-client.la
test_sdskv_migrate_test_LDFLAGS = -Llib -lsdskv-client
test_sdskv_multi_test_SOURCES = test/sdskv-multi-test.cc
test_sdskv_multi_test_DEPENDENCIES = lib/libsdskv-client.la
test_sdskv_multi_test_LDFLAGS = -Llib -lsdskv-client
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = maint/sdskv-server.pc \
maint/sdskv-client.pc
......
......@@ -104,6 +104,26 @@ int sdskv_put(sdskv_provider_handle_t provider,
const void *key, hg_size_t ksize,
const void *value, hg_size_t vsize);
/**
* @brief Puts multiple key/value pairs into the database.
* This method will send all the key/value pairs in batch,
* thus optimizing transfers by avoiding many RPC round trips.
*
* @param provider provider handle managing the database
* @param db_id targeted database id
* @param num number of key/value pairs to put
* @param keys array of keys
* @param ksizes array of key sizes
* @param values array of values
* @param vsizes array of value sizes
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
int sdskv_put_multi(sdskv_provider_handle_t provider,
sdskv_database_id_t db_id,
size_t num, const void** keys, const hg_size_t* ksizes,
const void** values, const hg_size_t *vsizes);
/**
* @brief Gets the value associated with a given key.
* vsize needs to be set to the current size of the allocated
......@@ -125,6 +145,34 @@ int sdskv_get(sdskv_provider_handle_t provider,
const void *key, hg_size_t ksize,
void *value, hg_size_t* vsize);
/**
* @brief Gets multiple values from the database. The transfers
* will be performed in a single batch. The vsize array should
* initially contain the size of the buffer allocated to receive
* each value. After a successful call, this array will contain
* the actual sizes of the values received. Contrary to sdskv_get,
* this function will not produce an error if one of the keys
* does not exist or if an allocated buffer is too small to hold
* a given value: the corresponding value entry will simply not
* be filled and its size will be set to 0 (so users must have
* another way to distinguish a 0-sized value and a value for
* which there was an error).
*
* @param[in] provider provider handle
* @param[in] db_id database id
* @param[in] num number of keys to retrieve
* @param[in] keys array of keys to retrieve
* @param[in] ksizes size of the keys
* @param[out] values array of allocated memory segments to receive the keys
* @param[inout] vsizes sizes allocated (in) and actual sizes (out)
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
int sdskv_get_multi(sdskv_provider_handle_t provider,
sdskv_database_id_t db_id,
size_t num, const void** keys, const hg_size_t* ksizes,
void** values, hg_size_t *vsizes);
/**
* @brief Gets the length of a value associated with a given key.
*
......@@ -140,6 +188,26 @@ int sdskv_length(sdskv_provider_handle_t handle,
sdskv_database_id_t db_id, const void *key,
hg_size_t ksize, hg_size_t* vsize);
/**
* @brief Gets the length of values associated with multiple keys.
* If a particular key does not exists, this function will set the length
* of its value to 0 (so the user needs another way to differenciate
* between a key that does not exists and a 0-sized value).
*
* @param[in] handle provider handle
* @param[in] db_id database id
* @param[in] num number of keys
* @param[in] keys array of keys
* @param[in] ksizes array of key sizes
* @param[out] vsizes array where to put value sizes
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
int sdskv_length_multi(sdskv_provider_handle_t handle,
sdskv_database_id_t db_id, size_t num,
const void** keys, const hg_size_t* ksizes,
hg_size_t *vsizes);
/**
* @brief Checks if the given key exists in the database.
*
......@@ -304,7 +372,7 @@ int sdskv_list_keyvals_with_prefix(
* @param num_keys number of keys
* @param keys array of keys
* @param key_sizes array of key sizes
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_BEFORE, or SDSKV_REMOVE_AFTER
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_ORIGINAL
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
......@@ -330,7 +398,7 @@ int sdskv_migrate_keys(
* @param target_db_id target database id
* @param key key to migrate
* @param key_size size of the key
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_BEFORE, or SDSKV_REMOVE_AFTER
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_ORIGINAL
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
......@@ -362,8 +430,8 @@ inline int sdskv_migrate_key(
* expressed by the array key_range, which contains two elements.
* key_range[0] must be a lower bound lb.
* key_range[1] must be an upper bound ub.
* The set of keys migrated are within the range [lb, ub[ (i.e. lb
* included, ub is not included).
* The set of keys migrated are within the range ]lb, ub[ (i.e. lb
* and ub not included).
*
* @param source_provider source provider
* @param source_db_id source database id
......@@ -372,7 +440,7 @@ inline int sdskv_migrate_key(
* @param target_db_id target database id
* @param key_range range of keys to migrate
* @param key_range_sizes size of the keys provided for the range
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_BEFORE, or SDSKV_REMOVE_AFTER
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_ORIGINAL
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
......@@ -398,7 +466,7 @@ int sdskv_migrate_key_range(
* @param target_db_id target database id
* @param key_prefix prefix of keys to migrate
* @param key_prefix_size size of the prefix provided
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_BEFORE, or SDSKV_REMOVE_AFTER
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_ORIGINAL
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
......@@ -421,7 +489,7 @@ int sdskv_migrate_keys_prefixed(
* @param target_addr target address
* @param target_provider_id target provider id
* @param target_db_id target database id
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_BEFORE, or SDSKV_REMOVE_AFTER
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_ORIGINAL
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
......@@ -433,33 +501,6 @@ int sdskv_migrate_all_keys(
sdskv_database_id_t target_db_id,
int flag);
/**
* @brief Migrates a database from a source provider
* to a target provider. The difference with sdskv_migrate_all_keys is
* that the target database does not exist yet and the id of the newly
* created database will be returned to the called.
* Contrary to sdskv_migrate_all_keys, if SDSKV_REMOVE_BEFORE or
* SDSKV_REMOVE_AFTER are used as flag, the source database is deleted
* on its provider (while sdskv_migrate_all_keys only removes all the keys,
* leaving the database present).
*
* @param source_provider source provider
* @param source_db_id source database id
* @param target_addr target address
* @param target_provider_id target provider id
* @param target_db_id resulting target database id
* @param flag SDSKV_KEEP_ORIGINAL, or SDSKV_REMOVE_BEFORE, or SDSKV_REMOVE_AFTER
*
* @return SDSKV_SUCCESS or error code defined in sdskv-common.h
*/
int sdskv_migrate_database(
sdskv_provider_handle_t source_provider,
sdskv_database_id_t source_db_id,
const char* target_addr,
uint16_t target_provider_id,
sdskv_database_id_t* target_db_id,
int flag);
/**
* Shuts down a remote SDSKV service (given an address).
* This will shutdown all the providers on the target address.
......
This diff is collapsed.
......@@ -113,6 +113,35 @@ MERCURY_GEN_PROC(bulk_get_in_t, ((uint64_t)(db_id))\
((hg_bulk_t)(handle)))
MERCURY_GEN_PROC(bulk_get_out_t, ((hg_size_t)(size)) ((int32_t)(ret)))
// ------------- PUT MULTI ------------- //
MERCURY_GEN_PROC(put_multi_in_t, \
((uint64_t)(db_id))\
((hg_size_t)(num_keys))\
((hg_bulk_t)(keys_bulk_handle))\
((hg_size_t)(keys_bulk_size))\
((hg_bulk_t)(vals_bulk_handle))\
((hg_size_t)(vals_bulk_size)))
MERCURY_GEN_PROC(put_multi_out_t, ((int32_t)(ret)))
// ------------- GET MULTI ------------- //
MERCURY_GEN_PROC(get_multi_in_t, \
((uint64_t)(db_id))\
((hg_size_t)(num_keys))\
((hg_bulk_t)(keys_bulk_handle))\
((hg_size_t)(keys_bulk_size))\
((hg_bulk_t)(vals_bulk_handle))\
((hg_size_t)(vals_bulk_size)))
MERCURY_GEN_PROC(get_multi_out_t, ((int32_t)(ret)))
// ------------- LENGTH MULTI ------------- //
MERCURY_GEN_PROC(length_multi_in_t, \
((uint64_t)(db_id))\
((hg_size_t)(num_keys))\
((hg_bulk_t)(keys_bulk_handle))\
((hg_size_t)(keys_bulk_size))\
((hg_bulk_t)(vals_size_bulk_handle)))
MERCURY_GEN_PROC(length_multi_out_t, ((int32_t)(ret)))
// ------------- MIGRATE KEYS ----------- //
MERCURY_GEN_PROC(migrate_keys_in_t,
((uint64_t)(source_db_id))\
......@@ -154,15 +183,5 @@ MERCURY_GEN_PROC(migrate_all_keys_in_t,
((uint64_t)(target_db_id))\
((int32_t)(flag)))
// ------------- MIGRATE DATABASE ----------- //
MERCURY_GEN_PROC(migrate_database_in_t,
((uint64_t)(source_db_id))\
((hg_string_t)(target_addr))\
((uint16_t)(target_provider_id))\
((int32_t)(flag)))
MERCURY_GEN_PROC(migrate_database_out_t,
((int32_t)(ret))\
((uint64_t)(db_id)))
#endif
This diff is collapsed.
#!/bin/bash -x
if [ -z $srcdir ]; then
echo srcdir variable not set.
exit 1
fi
source $srcdir/test/test-util.sh
find_db_name
# start a server with 2 second wait,
# 20s timeout, and my_test_db as database
test_start_server 2 20 $test_db_full
sleep 1
#####################
run_to 20 test/sdskv-multi-test $svr_addr 1 $test_db_name 100
if [ $? -ne 0 ]; then
wait
exit 1
fi
wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
exit 0
/*
* (C) 2015 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#include <stdio.h>
#include <assert.h>
#include <unistd.h>
#include <margo.h>
#include <iostream>
#include <string>
#include <vector>
#include <algorithm>
#include <map>
#include "sdskv-client.h"
static std::string gen_random_string(size_t len);
int main(int argc, char *argv[])
{
char cli_addr_prefix[64] = {0};
char *sdskv_svr_addr_str;
char *db_name;
margo_instance_id mid;
hg_addr_t svr_addr;
uint8_t mplex_id;
uint32_t num_keys;
sdskv_client_t kvcl;
sdskv_provider_handle_t kvph;
hg_return_t hret;
int ret;
if(argc != 5)
{
fprintf(stderr, "Usage: %s <sdskv_server_addr> <mplex_id> <db_name> <num_keys>\n", argv[0]);
fprintf(stderr, " Example: %s tcp://localhost:1234 1 foo 1000\n", argv[0]);
return(-1);
}
sdskv_svr_addr_str = argv[1];
mplex_id = atoi(argv[2]);
db_name = argv[3];
num_keys = atoi(argv[4]);
/* initialize Margo using the transport portion of the server
* address (i.e., the part before the first : character if present)
*/
for(unsigned i=0; (i<63 && sdskv_svr_addr_str[i] != '\0' && sdskv_svr_addr_str[i] != ':'); i++)
cli_addr_prefix[i] = sdskv_svr_addr_str[i];
/* start margo */
mid = margo_init(cli_addr_prefix, MARGO_SERVER_MODE, 0, 0);
if(mid == MARGO_INSTANCE_NULL)
{
fprintf(stderr, "Error: margo_init()\n");
return(-1);
}
ret = sdskv_client_init(mid, &kvcl);
if(ret != 0)
{
fprintf(stderr, "Error: sdskv_client_init()\n");
margo_finalize(mid);
return -1;
}
/* look up the SDSKV server address */
hret = margo_addr_lookup(mid, sdskv_svr_addr_str, &svr_addr);
if(hret != HG_SUCCESS)
{
fprintf(stderr, "Error: margo_addr_lookup()\n");
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(-1);
}
/* create a SDSKV provider handle */
ret = sdskv_provider_handle_create(kvcl, svr_addr, mplex_id, &kvph);
if(ret != 0)
{
fprintf(stderr, "Error: sdskv_provider_handle_create()\n");
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(-1);
}
/* open the database */
sdskv_database_id_t db_id;
ret = sdskv_open(kvph, db_name, &db_id);
if(ret == 0) {
printf("Successfuly open database %s, id is %ld\n", db_name, db_id);
} else {
fprintf(stderr, "Error: could not open database %s\n", db_name);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(-1);
}
/* **** generate multiple key/vals ***** */
std::vector<std::string> keys;
std::vector<std::string> vals;
std::map<std::string, std::string> reference;
size_t max_value_size = 24;
for(unsigned i=0; i < num_keys; i++) {
auto k = gen_random_string(16);
auto v = gen_random_string(3+i*(max_value_size-3)/num_keys);
reference[k] = v;
keys.push_back(k);
vals.push_back(v);
}
std::vector<const void*> keys_ptr(keys.size());
std::vector<const void*> vals_ptr(vals.size());
std::vector<hg_size_t> keys_size(keys.size());
std::vector<hg_size_t> vals_size(vals.size());
for(unsigned i=0; i < num_keys; i++) {
keys_ptr[i] = (const void*)keys[i].data();
vals_ptr[i] = (const void*)vals[i].data();
keys_size[i] = keys[i].size()+1; // +1 because of the null character
vals_size[i] = vals[i].size()+1;
}
/* **** issue a put_multi ***** */
ret = sdskv_put_multi(kvph, db_id, num_keys, &keys_ptr[0], keys_size.data(),
&vals_ptr[0], vals_size.data());
if(ret != 0) {
fprintf(stderr, "Error: sdskv_put_multi() failed\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
printf("Successfuly inserted %d keys\n", num_keys);
/* retrieve the length of the values */
std::vector<hg_size_t> rval_len(num_keys);
ret = sdskv_length_multi(kvph, db_id, num_keys,
keys_ptr.data(), keys_size.data(), rval_len.data());
if(ret != 0) {
fprintf(stderr, "Error: sdskv_length_multi() failed\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
/* check if the lengths are correct */
for(unsigned i=0; i < num_keys; i++) {
if(rval_len[i] != vals_size[i]) {
fprintf(stderr, "Error: value %d doesn't have the right length (%ld != %ld)\n", i,
rval_len[i], vals_size[i]);
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
}
/* **** get keys **** */
std::vector<std::vector<char>> read_values(num_keys);
for(unsigned i=0; i < num_keys; i++) {
read_values[i].resize(rval_len[i]);
}
std::vector<void*> read_values_ptr(num_keys);
for(unsigned i=0; i < num_keys; i++) {
read_values_ptr[i] = read_values[i].data();
}
ret = sdskv_get_multi(kvph, db_id, num_keys,
keys_ptr.data(), keys_size.data(),
read_values_ptr.data(), rval_len.data());
if(ret != 0) {
fprintf(stderr, "Error: sdskv_get_multi() failed\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
/* check the keys we received against reference */
for(unsigned i=0; i < num_keys; i++) {
std::string vstring(read_values[i].data());
auto& k = keys[i];
std::cout << "Got " << k << " ===> " << vstring << "\t" << "expected: " << reference[k] << std::endl;
if(vstring != reference[k]) {
fprintf(stderr, "Error: sdskv_get_multi() returned a value different from the reference\n");
sdskv_shutdown_service(kvcl, svr_addr);
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return -1;
}
}
/* shutdown the server */
ret = sdskv_shutdown_service(kvcl, svr_addr);
/**** cleanup ****/
sdskv_provider_handle_release(kvph);
margo_addr_free(mid, svr_addr);
sdskv_client_finalize(kvcl);
margo_finalize(mid);
return(ret);
}
static std::string gen_random_string(size_t len) {
static const char alphanum[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
std::string s(len, ' ');
for (unsigned i = 0; i < len; ++i) {
s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];
}
return s;
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment