Commit 5f8918aa authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

added prefix feature to BDB and LDB, and tested

parent 30801a72
......@@ -102,8 +102,8 @@ AC_ARG_ENABLE([leveldb],
)
AC_ARG_ENABLE([bwtree],
AS_HELP_STRING([--enable-bwtree], [Enable BwTree as server
backend. (default is yes) ]),
[bwtree_backend=${enableval}], [bwtree_backend=yes]
backend. (default is no) ]),
[bwtree_backend=${enableval}], [bwtree_backend=no]
)
......@@ -129,6 +129,7 @@ fi
if test "x${bwtree_backend}" == xyes ; then
AC_DEFINE([USE_BWTREE], 1, [use BwTree backend])
AC_MSG_WARN([BwTree backend is deprecated])
CPPFLAGS="-I${srcdir}/src/BwTree/src ${CPPFLAGS}"
CXXFLAGS="-pthread -std=c++11 -g -Wall -mcx16 -Wno-invalid-offsetof ${CXXFLAGS}"
fi
......
......@@ -224,50 +224,94 @@ std::vector<ds_bulk_t> BerkeleyDBDataStore::vlist_keys(const ds_bulk_t &start, s
/* 'start' is like RADOS: not inclusive */
if (start.size()) {
key.set_size(start.size());
key.set_data((void *)start.data());
ret = cursorp->get(&key, &data, DB_SET_RANGE);
if (ret != 0) {
cursorp->close();
return keys;
}
ds_bulk_t k(key.get_size() );
memcpy(k.data(), key.get_data(), key.get_size() );
key.set_size(start.size());
key.set_data((void *)start.data());
ret = cursorp->get(&key, &data, DB_SET_RANGE);
if (ret != 0) {
cursorp->close();
return keys;
}
} else {
ret = cursorp->get(&key, &data, DB_FIRST);
if (ret != 0) {
cursorp->close();
return keys;
}
}
ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
/* SET_RANGE will return the smallest key greater than or equal to the
* requested key, but we want strictly greater than */
if (k != start)
keys.push_back(std::move(k));
int c = 0;
if (k != start) {
c = std::memcmp(prefix.data(), k.data(), prefix.size());
if(c == 0) {
keys.push_back(std::move(k));
}
}
while (keys.size() < count) {
ret = cursorp->get(&key, &data, DB_NEXT);
if (ret !=0 ) break;
ds_bulk_t k(key.get_size() );
memcpy(k.data(), key.get_data(), key.get_size() );
keys.push_back(std::move(k));
while (keys.size() < count && c >= 0) {
ret = cursorp->get(&key, &data, DB_NEXT);
if (ret !=0 ) break;
ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
c = std::memcmp(prefix.data(), k.data(), prefix.size());
if(c == 0) {
keys.push_back(std::move(k));
}
}
cursorp->close();
return keys;
}
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::vlist_keyvals(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
{
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;
Dbc * cursorp;
Dbt key, data;
int ret;
_dbm->cursor(NULL, &cursorp, 0);
for (size_t i=0; i< count; i++) {
int ret = cursorp->get(&key, &data, DB_NEXT);
if (ret !=0 ) break;
ds_bulk_t k(key.get_size());
ds_bulk_t v(data.get_size());
/* 'start' is like RADOS: not inclusive */
if (start.size()) {
key.set_size(start.size());
key.set_data((void *)start.data());
ret = cursorp->get(&key, &data, DB_SET_RANGE);
if (ret != 0) {
cursorp->close();
return result;
}
} else {
ret = cursorp->get(&key, &data, DB_FIRST);
if (ret != 0) {
cursorp->close();
return result;
}
}
ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
ds_bulk_t v((char*)data.get_data(), ((char*)data.get_data())+data.get_size());
memcpy(k.data(), key.get_data(), key.get_size());
memcpy(v.data(), data.get_data(), data.get_size());
/* SET_RANGE will return the smallest key greater than or equal to the
* requested key, but we want strictly greater than */
int c = 0;
if (k != start) {
c = std::memcmp(prefix.data(), k.data(), prefix.size());
if(c == 0) {
result.push_back(std::make_pair(std::move(k),std::move(v)));
}
}
while (result.size() < count && c >= 0) {
ret = cursorp->get(&key, &data, DB_NEXT);
if (ret !=0 ) break;
ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
ds_bulk_t v((char*)data.get_data(), ((char*)data.get_data())+data.get_size());
keyvals.push_back(std::make_pair(std::move(k), std::move(v)));
c = std::memcmp(prefix.data(), k.data(), prefix.size());
if(c == 0) {
result.push_back(std::make_pair(std::move(k), std::move(v)));
}
}
cursorp->close();
return keyvals;
return result;
}
......@@ -134,28 +134,72 @@ std::vector<ds_bulk_t> LevelDBDataStore::vlist_keys(const ds_bulk_t &start, size
leveldb::Iterator *it = _dbm->NewIterator(leveldb::ReadOptions());
leveldb::Slice start_slice(start.data(), start.size());
int c = 0;
if (start.size() > 0) {
it->Seek(start_slice);
/* we treat 'start' the way RADOS treats it: excluding it from returned
* keys. LevelDB treats start inclusively, so skip over it if we found
* an exact match */
if ( start.size() == it->key().size() &&
(memcmp(it->key().data(), start.data(), start.size()) == 0))
it->Next();
it->Seek(start_slice);
/* we treat 'start' the way RADOS treats it: excluding it from returned
* keys. LevelDB treats start inclusively, so skip over it if we found
* an exact match */
if ( start.size() == it->key().size() &&
(memcmp(it->key().data(), start.data(), start.size()) == 0))
it->Next();
} else {
it->SeekToFirst();
it->SeekToFirst();
}
/* note: iterator initialized above, not in for loop */
for (; it->Valid() && keys.size() < count; it->Next() ) {
ds_bulk_t k(it->key().size());
memcpy(k.data(), it->key().data(), it->key().size() );
keys.push_back(std::move(k));
c = std::memcmp(prefix.data(), k.data(), prefix.size());
if(c == 0) {
keys.push_back(std::move(k));
} else if(c < 0) {
break;
}
}
delete it;
return keys;
}
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
{
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;
leveldb::Iterator *it = _dbm->NewIterator(leveldb::ReadOptions());
leveldb::Slice start_slice(start.data(), start.size());
int c = 0;
if (start.size() > 0) {
it->Seek(start_slice);
/* we treat 'start' the way RADOS treats it: excluding it from returned
* keys. LevelDB treats start inclusively, so skip over it if we found
* an exact match */
if ( start.size() == it->key().size() &&
(memcmp(it->key().data(), start.data(), start.size()) == 0))
it->Next();
} else {
it->SeekToFirst();
}
/* note: iterator initialized above, not in for loop */
for (; it->Valid() && result.size() < count; it->Next() ) {
ds_bulk_t k(it->key().size());
ds_bulk_t v(it->value().size());
memcpy(k.data(), it->key().data(), it->key().size());
memcpy(v.data(), it->value().data(), it->value().size());
c = std::memcmp(prefix.data(), k.data(), prefix.size());
if(c == 0) {
result.push_back(std::make_pair(std::move(k), std::move(v)));
} else if(c < 0) {
break;
}
}
delete it;
return result;
}
/*
{
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
......@@ -172,3 +216,4 @@ std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(cons
delete it;
return keyvals;
}
*/
......@@ -28,7 +28,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -27,7 +27,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -27,7 +27,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -27,7 +27,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -27,7 +27,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -27,7 +27,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -16,7 +16,7 @@ sleep 1
#####################
run_to 20 test/sdskv-list-keyvals-test $svr_addr 1 $test_db_name 10
run_to 20 test/sdskv-list-keyvals-test $svr_addr 1 $test_db_name 30
if [ $? -ne 0 ]; then
wait
exit 1
......@@ -27,7 +27,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -28,7 +28,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -27,7 +27,4 @@ wait
echo cleaning up $TMPBASE
rm -rf $TMPBASE
echo cleaning up $test_db_name
rm_db
exit 0
......@@ -8,6 +8,7 @@
#include <assert.h>
#include <unistd.h>
#include <margo.h>
#include <iostream>
#include <string>
#include <vector>
#include <algorithm>
......@@ -102,12 +103,12 @@ int main(int argc, char *argv[])
/* **** put keys ***** */
std::vector<std::string> keys;
std::map<std::string, std::string> reference;
size_t max_value_size = 8000;
size_t max_value_size = 24;
for(unsigned i=0; i < num_keys; i++) {
auto k = gen_random_string(16);
// half of the entries will be put using bulk
auto v = gen_random_string(i*max_value_size/num_keys);
auto v = gen_random_string(3+i*(max_value_size-3)/num_keys);
ret = sdskv_put(kvph, db_id,
(const void *)k.data(), k.size()+1,
(const void *)v.data(), v.size()+1);
......@@ -120,6 +121,7 @@ int main(int argc, char *argv[])
margo_finalize(mid);
return -1;
}
std::cout << "Inserted " << k << "\t ===> " << v << std::endl;
reference[k] = v;
keys.push_back(k);
}
......@@ -143,6 +145,7 @@ int main(int argc, char *argv[])
return -1;
}
std::string vstring((char*)(v.data()));
std::cout << "Got " << k << " ===> " << vstring << std::endl;
if(vstring != reference[k]) {
fprintf(stderr, "Error: sdskv_get() returned a value different from the reference\n");
sdskv_shutdown_service(kvcl, svr_addr);
......
......@@ -103,7 +103,7 @@ int main(int argc, char *argv[])
/* **** put keys ***** */
std::vector<std::string> keys;
std::map<std::string, std::string> reference;
size_t max_value_size = 16;
size_t max_value_size = 8000;
size_t max_key_size = 16;
for(unsigned i=0; i < num_keys; i++) {
......
......@@ -44,12 +44,7 @@ function test_start_server ()
function find_db_name ()
{
test_db_name=${SDSKV_TEST_DB_NAME:-"/tmp/sdskv-test-db"}
test_db_name=$TMPBASE/${SDSKV_TEST_DB_NAME:-"sdskv-test-db"}
test_db_type=${SDSKV_TEST_DB_TYPE:-"map"}
test_db_full="${test_db_name}:${test_db_type}"
}
function rm_db()
{
rm -rf ${test_db_name}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment