added prefix feature to BDB and LDB, and tested

configure.ac

@@ -102,8 +102,8 @@ AC_ARG_ENABLE([leveldb],
 )
 AC_ARG_ENABLE([bwtree],
 	     AS_HELP_STRING([--enable-bwtree], [Enable BwTree as server
-			     backend.  (default is yes) ]),
-	     [bwtree_backend=${enableval}], [bwtree_backend=yes]
+			     backend.  (default is no) ]),
+	     [bwtree_backend=${enableval}], [bwtree_backend=no]
 )
 
 
@@ -129,6 +129,7 @@ fi
 
 if test "x${bwtree_backend}" == xyes ; then
 	AC_DEFINE([USE_BWTREE], 1, [use BwTree backend])
+	AC_MSG_WARN([BwTree backend is deprecated])
 	CPPFLAGS="-I${srcdir}/src/BwTree/src ${CPPFLAGS}"
 	CXXFLAGS="-pthread -std=c++11 -g -Wall -mcx16 -Wno-invalid-offsetof ${CXXFLAGS}"
 fi

src/datastore/berkeleydb_datastore.cc

@@ -224,50 +224,94 @@ std::vector<ds_bulk_t> BerkeleyDBDataStore::vlist_keys(const ds_bulk_t &start, s
 
     /* 'start' is like RADOS: not inclusive  */
     if (start.size()) {
-	key.set_size(start.size());
-	key.set_data((void *)start.data());
-	ret = cursorp->get(&key, &data, DB_SET_RANGE);
-	if (ret != 0) {
-	    cursorp->close();
-	    return keys;
-	}
-	ds_bulk_t k(key.get_size() );
-	memcpy(k.data(), key.get_data(), key.get_size() );
+	    key.set_size(start.size());
+	    key.set_data((void *)start.data());
+	    ret = cursorp->get(&key, &data, DB_SET_RANGE);
+	    if (ret != 0) {
+	        cursorp->close();
+	        return keys;
+	    }
+    } else {
+        ret = cursorp->get(&key, &data, DB_FIRST);
+        if (ret != 0) {
+            cursorp->close();
+            return keys;
+        }
+    }
+
+	ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
 	/* SET_RANGE will return the smallest key greater than or equal to the
 	 * requested key, but we want strictly greater than */
-	if (k != start)
-	    keys.push_back(std::move(k));
+    int c = 0;
+	if (k != start) {
+        c = std::memcmp(prefix.data(), k.data(), prefix.size());
+        if(c == 0) {
+            keys.push_back(std::move(k));
+        }
     }
-    while (keys.size() < count) {
-	ret = cursorp->get(&key, &data, DB_NEXT);
-	if (ret !=0 ) break;
-
-	ds_bulk_t k(key.get_size() );
-	memcpy(k.data(), key.get_data(), key.get_size() );
-	keys.push_back(std::move(k));
+    while (keys.size() < count && c >= 0) {
+	    ret = cursorp->get(&key, &data, DB_NEXT);
+	    if (ret !=0 ) break;
+        
+        ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
+        c = std::memcmp(prefix.data(), k.data(), prefix.size());
+        if(c == 0) {
+            keys.push_back(std::move(k));
+        }
     }
     cursorp->close();
     return keys;
 }
 
-std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
+std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::vlist_keyvals(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
 {
-    std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
+    std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;
     Dbc * cursorp;
     Dbt key, data;
+    int ret;
     _dbm->cursor(NULL, &cursorp, 0);
-    for (size_t i=0; i< count; i++) {
-        int ret = cursorp->get(&key, &data, DB_NEXT);
-        if (ret !=0 ) break;
 
-        ds_bulk_t k(key.get_size());
-        ds_bulk_t v(data.get_size());
+    /* 'start' is like RADOS: not inclusive  */
+    if (start.size()) {
+	    key.set_size(start.size());
+	    key.set_data((void *)start.data());
+	    ret = cursorp->get(&key, &data, DB_SET_RANGE);
+	    if (ret != 0) {
+	        cursorp->close();
+	        return result;
+	    }
+    } else {
+        ret = cursorp->get(&key, &data, DB_FIRST);
+        if (ret != 0) {
+            cursorp->close();
+            return result;
+        }
+    }
+
+	ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
+    ds_bulk_t v((char*)data.get_data(), ((char*)data.get_data())+data.get_size());
 
-        memcpy(k.data(), key.get_data(), key.get_size());
-        memcpy(v.data(), data.get_data(), data.get_size());
+	/* SET_RANGE will return the smallest key greater than or equal to the
+	 * requested key, but we want strictly greater than */
+    int c = 0;
+	if (k != start) {
+        c = std::memcmp(prefix.data(), k.data(), prefix.size());
+        if(c == 0) {
+            result.push_back(std::make_pair(std::move(k),std::move(v)));
+        }
+    }
+    while (result.size() < count && c >= 0) {
+	    ret = cursorp->get(&key, &data, DB_NEXT);
+	    if (ret !=0 ) break;
+        
+        ds_bulk_t k((char*)key.get_data(), ((char*)key.get_data())+key.get_size());
+        ds_bulk_t v((char*)data.get_data(), ((char*)data.get_data())+data.get_size());
 
-        keyvals.push_back(std::make_pair(std::move(k), std::move(v)));
+        c = std::memcmp(prefix.data(), k.data(), prefix.size());
+        if(c == 0) {
+            result.push_back(std::make_pair(std::move(k), std::move(v)));
+        }
     }
     cursorp->close();
-    return keyvals;
+    return result;
 }

src/datastore/leveldb_datastore.cc

@@ -134,28 +134,72 @@ std::vector<ds_bulk_t> LevelDBDataStore::vlist_keys(const ds_bulk_t &start, size
     leveldb::Iterator *it = _dbm->NewIterator(leveldb::ReadOptions());
     leveldb::Slice start_slice(start.data(), start.size());
 
+    int c = 0;
+
     if (start.size() > 0) {
-	it->Seek(start_slice);
-	/* we treat 'start' the way RADOS treats it: excluding it from returned
-	 * keys. LevelDB treats start inclusively, so skip over it if we found
-	 * an exact match */
-	if ( start.size() == it->key().size() &&
-		(memcmp(it->key().data(), start.data(), start.size()) == 0))
-	    it->Next();
+        it->Seek(start_slice);
+        /* we treat 'start' the way RADOS treats it: excluding it from returned
+         * keys. LevelDB treats start inclusively, so skip over it if we found
+         * an exact match */
+        if ( start.size() == it->key().size() &&
+                (memcmp(it->key().data(), start.data(), start.size()) == 0))
+            it->Next();
     } else {
-	it->SeekToFirst();
+        it->SeekToFirst();
     }
     /* note: iterator initialized above, not in for loop */
     for (; it->Valid() && keys.size() < count; it->Next() ) {
         ds_bulk_t k(it->key().size());
         memcpy(k.data(), it->key().data(), it->key().size() );
-        keys.push_back(std::move(k));
+        c = std::memcmp(prefix.data(), k.data(), prefix.size());
+        if(c == 0) {
+            keys.push_back(std::move(k));
+        } else if(c < 0) {
+            break;
+        }
     }
     delete it;
     return keys;
 }
 
-std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
+std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
+{
+    std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;
+
+    leveldb::Iterator *it = _dbm->NewIterator(leveldb::ReadOptions());
+    leveldb::Slice start_slice(start.data(), start.size());
+
+    int c = 0;
+
+    if (start.size() > 0) {
+        it->Seek(start_slice);
+        /* we treat 'start' the way RADOS treats it: excluding it from returned
+         * keys. LevelDB treats start inclusively, so skip over it if we found
+         * an exact match */
+        if ( start.size() == it->key().size() &&
+                (memcmp(it->key().data(), start.data(), start.size()) == 0))
+            it->Next();
+    } else {
+        it->SeekToFirst();
+    }
+    /* note: iterator initialized above, not in for loop */
+    for (; it->Valid() && result.size() < count; it->Next() ) {
+        ds_bulk_t k(it->key().size());
+        ds_bulk_t v(it->value().size());
+        memcpy(k.data(), it->key().data(), it->key().size());
+        memcpy(v.data(), it->value().data(), it->value().size());
+
+        c = std::memcmp(prefix.data(), k.data(), prefix.size());
+        if(c == 0) {
+            result.push_back(std::make_pair(std::move(k), std::move(v)));
+        } else if(c < 0) {
+            break;
+        }
+    }
+    delete it;
+    return result;
+}
+/*
 {
     std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
 
@@ -172,3 +216,4 @@ std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(cons
     delete it;
     return keyvals;
 }
+*/

test/basic.sh

@@ -28,7 +28,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/erase-test.sh

@@ -27,7 +27,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/get-test.sh

@@ -27,7 +27,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/length-test.sh

@@ -27,7 +27,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/list-keys-prefix-test.sh

@@ -27,7 +27,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/list-keys-test.sh

@@ -27,7 +27,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/list-keyvals-test.sh

@@ -16,7 +16,7 @@ sleep 1
 
 #####################
 
-run_to 20 test/sdskv-list-keyvals-test $svr_addr 1 $test_db_name 10
+run_to 20 test/sdskv-list-keyvals-test $svr_addr 1 $test_db_name 30
 if [ $? -ne 0 ]; then
     wait
     exit 1
@@ -27,7 +27,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/open-test.sh

@@ -28,7 +28,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/put-test.sh

@@ -27,7 +27,4 @@ wait
 echo cleaning up $TMPBASE
 rm -rf $TMPBASE
 
-echo cleaning up $test_db_name
-rm_db
-
 exit 0

test/sdskv-get-test.cc

@@ -8,6 +8,7 @@
 #include <assert.h>
 #include <unistd.h>
 #include <margo.h>
+#include <iostream>
 #include <string>
 #include <vector>
 #include <algorithm>
@@ -102,12 +103,12 @@ int main(int argc, char *argv[])
     /* **** put keys ***** */
     std::vector<std::string> keys;
     std::map<std::string, std::string> reference;
-    size_t max_value_size = 8000;
+    size_t max_value_size = 24;
 
     for(unsigned i=0; i < num_keys; i++) {
         auto k = gen_random_string(16);
         // half of the entries will be put using bulk
-        auto v = gen_random_string(i*max_value_size/num_keys);
+        auto v = gen_random_string(3+i*(max_value_size-3)/num_keys);
         ret = sdskv_put(kvph, db_id,
                 (const void *)k.data(), k.size()+1,
                 (const void *)v.data(), v.size()+1);
@@ -120,6 +121,7 @@ int main(int argc, char *argv[])
             margo_finalize(mid);
             return -1;
         }
+        std::cout << "Inserted " << k << "\t ===> " << v << std::endl;
         reference[k] = v;
         keys.push_back(k);
     }
@@ -143,6 +145,7 @@ int main(int argc, char *argv[])
             return -1;
         }
         std::string vstring((char*)(v.data()));
+        std::cout << "Got " << k << " ===> " << vstring << std::endl;
         if(vstring != reference[k]) {
             fprintf(stderr, "Error: sdskv_get() returned a value different from the reference\n");
             sdskv_shutdown_service(kvcl, svr_addr);

test/sdskv-list-kv-test.cc

@@ -103,7 +103,7 @@ int main(int argc, char *argv[])
     /* **** put keys ***** */
     std::vector<std::string> keys;
     std::map<std::string, std::string> reference;
-    size_t max_value_size = 16;
+    size_t max_value_size = 8000;
     size_t max_key_size = 16;
 
     for(unsigned i=0; i < num_keys; i++) {

test/test-util.sh

@@ -44,12 +44,7 @@ function test_start_server ()
 
 function find_db_name ()
 {
-    test_db_name=${SDSKV_TEST_DB_NAME:-"/tmp/sdskv-test-db"}
+    test_db_name=$TMPBASE/${SDSKV_TEST_DB_NAME:-"sdskv-test-db"}
     test_db_type=${SDSKV_TEST_DB_TYPE:-"map"}
     test_db_full="${test_db_name}:${test_db_type}"
 }
-
-function rm_db()
-{
-    rm -rf ${test_db_name}
-}