list-keys test with prefix passing with map database

Makefile.am

@@ -13,7 +13,8 @@ bin_PROGRAMS = bin/sdskv-server-daemon 	    \
 	       test/sdskv-get-test 	    \
 	       test/sdskv-erase-test        \
 	       test/sdskv-list-keys-test    \
-	       test/sdskv-list-keyvals-test 
+	       test/sdskv-list-keyvals-test \
+	       test/sdskv-list-keys-prefix-test 
 
 bin_sdskv_server_daemon_SOURCES = src/sdskv-server-daemon.c
 bin_sdskv_server_daemon_DEPENDENCIES = lib/libsdskv-server.la
@@ -96,7 +97,8 @@ TESTS = test/basic.sh 		\
 	test/get-test.sh 	\
 	test/erase-test.sh      \
 	test/list-keys-test.sh  \
-	test/list-keyvals-test.sh 
+	test/list-keyvals-test.sh  \
+	test/list-keys-prefix-test.sh
 
 TESTS_ENVIRONMENT = TIMEOUT="$(TIMEOUT)" \
 		    MKTEMP="$(MKTEMP)"
@@ -121,6 +123,10 @@ test_sdskv_list_keys_test_SOURCES = test/sdskv-list-keys-test.cc
 test_sdskv_list_keys_test_DEPENDENCIES = lib/libsdskv-client.la
 test_sdskv_list_keys_test_LDFLAGS = -Llib -lsdskv-client
 
+test_sdskv_list_keys_prefix_test_SOURCES = test/sdskv-list-keys-prefix-test.cc
+test_sdskv_list_keys_prefix_test_DEPENDENCIES = lib/libsdskv-client.la
+test_sdskv_list_keys_prefix_test_LDFLAGS = -Llib -lsdskv-client
+
 test_sdskv_list_keyvals_test_SOURCES = test/sdskv-list-kv-test.cc
 test_sdskv_list_keyvals_test_DEPENDENCIES = lib/libsdskv-client.la
 test_sdskv_list_keyvals_test_LDFLAGS = -Llib -lsdskv-client

src/datastore/map_datastore.h

@@ -60,12 +60,23 @@ class MapDataStore : public AbstractDataStore {
         virtual std::vector<ds_bulk_t> vlist_keys(
                 const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix) {
             std::vector<ds_bulk_t> result;
-            auto it = _map.lower_bound(start_key);
-            while(it != _map.end() && it->first == start_key) {
-                it++;
+            decltype(_map.begin()) it;
+            if(start_key.size() > 0) {
+                it = _map.lower_bound(start_key);
+                while(it != _map.end() && it->first == start_key) it++;
+            } else {
+                it = _map.begin();
             }
-            for(size_t i=0; it != _map.end() && i < count; it++, i++) {
-                result.push_back(it->first);
+            while(result.size() < count && it != _map.end()) {
+                const auto& p = *it;
+                if(prefix.size() > p.first.size()) continue;
+                int c = std::memcmp(prefix.data(), p.first.data(), prefix.size());
+                if(c == 0) {
+                    result.push_back(p.first);
+                } else if(c < 0) {
+                    break; // we have exceeded prefix
+                }
+                it++;
             }
             return result;
         }
@@ -85,7 +96,7 @@ class MapDataStore : public AbstractDataStore {
                 if(prefix.size() > p.first.size()) continue;
                 int c = std::memcmp(prefix.data(), p.first.data(), prefix.size());
                 if(c == 0) {
-                    result.push_back(*it);
+                    result.push_back(p);
                 } else if(c < 0) {
                     break; // we have exceeded prefix
                 }

test/list-keys-prefix-test.sh

+#!/bin/bash -x
+
+if [ -z $srcdir ]; then
+    echo srcdir variable not set.
+    exit 1
+fi
+source $srcdir/test/test-util.sh
+
+find_db_name
+
+# start a server with 2 second wait,
+# 20s timeout, and my_test_db as database
+test_start_server 2 20 $test_db_full
+
+sleep 1
+
+#####################
+
+run_to 20 test/sdskv-list-keys-prefix-test $svr_addr 1 $test_db_name 10
+if [ $? -ne 0 ]; then
+    wait
+    exit 1
+fi
+
+wait
+
+echo cleaning up $TMPBASE
+rm -rf $TMPBASE
+
+echo cleaning up $test_db_name
+rm_db
+
+exit 0

test/sdskv-list-keys-prefix-test.cc

+/*
+ * (C) 2015 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
+
+#include <stdio.h>
+#include <assert.h>
+#include <unistd.h>
+#include <margo.h>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <map>
+
+#include "sdskv-client.h"
+
+static std::string gen_random_string(size_t len);
+
+int main(int argc, char *argv[])
+{
+    char cli_addr_prefix[64] = {0};
+    char *sdskv_svr_addr_str;
+    char *db_name;
+    margo_instance_id mid;
+    hg_addr_t svr_addr;
+    uint8_t mplex_id;
+    uint32_t num_keys;
+    sdskv_client_t kvcl;
+    sdskv_provider_handle_t kvph;
+    hg_return_t hret;
+    int ret;
+
+    if(argc != 5)
+    {
+        fprintf(stderr, "Usage: %s <sdskv_server_addr> <mplex_id> <db_name> <num_keys>\n", argv[0]);
+        fprintf(stderr, "  Example: %s tcp://localhost:1234 1 foo 1000\n", argv[0]);
+        return(-1);
+    }
+    sdskv_svr_addr_str = argv[1];
+    mplex_id           = atoi(argv[2]);
+    db_name            = argv[3];
+    num_keys           = atoi(argv[4]);
+
+    /* initialize Margo using the transport portion of the server
+     * address (i.e., the part before the first : character if present)
+     */
+    for(unsigned i=0; (i<63 && sdskv_svr_addr_str[i] != '\0' && sdskv_svr_addr_str[i] != ':'); i++)
+        cli_addr_prefix[i] = sdskv_svr_addr_str[i];
+
+    /* start margo */
+    mid = margo_init(cli_addr_prefix, MARGO_SERVER_MODE, 0, 0);
+    if(mid == MARGO_INSTANCE_NULL)
+    {
+        fprintf(stderr, "Error: margo_init()\n");
+        return(-1);
+    }
+
+    ret = sdskv_client_init(mid, &kvcl);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: sdskv_client_init()\n");
+        margo_finalize(mid);
+        return -1;
+    }
+
+    /* look up the SDSKV server address */
+    hret = margo_addr_lookup(mid, sdskv_svr_addr_str, &svr_addr);
+    if(hret != HG_SUCCESS)
+    {
+        fprintf(stderr, "Error: margo_addr_lookup()\n");
+        sdskv_client_finalize(kvcl);
+        margo_finalize(mid);
+        return(-1);
+    }
+
+    /* create a SDSKV provider handle */
+    ret = sdskv_provider_handle_create(kvcl, svr_addr, mplex_id, &kvph);
+    if(ret != 0)
+    {
+        fprintf(stderr, "Error: sdskv_provider_handle_create()\n");
+        margo_addr_free(mid, svr_addr);
+        sdskv_client_finalize(kvcl);
+        margo_finalize(mid);
+        return(-1);
+    }
+
+    /* open the database */
+    sdskv_database_id_t db_id;
+    ret = sdskv_open(kvph, db_name, &db_id);
+    if(ret == 0) {
+        printf("Successfuly open database %s, id is %ld\n", db_name, db_id);
+    } else {
+        fprintf(stderr, "Error: could not open database %s\n", db_name);
+        sdskv_provider_handle_release(kvph);
+        margo_addr_free(mid, svr_addr);
+        sdskv_client_finalize(kvcl);
+        margo_finalize(mid);
+        return(-1);
+    }
+
+    /* **** put keys ***** */
+    std::vector<std::string> keys;
+    std::string prefix("XYZ");
+    size_t max_key_size   = 16;
+    size_t max_value_size = 16;
+
+    for(unsigned i=0; i < num_keys; i++) {
+        std::string k;
+        if(i % 2 == 0) {
+            k = prefix;
+            k += gen_random_string(((max_key_size + (rand()%max_key_size))/2) - prefix.size());
+        } else {
+            k = gen_random_string(((max_key_size + (rand()%max_key_size))/2));
+        }
+        // half of the entries will be put using bulk
+        auto v = gen_random_string(i*max_value_size/num_keys);
+        ret = sdskv_put(kvph, db_id,
+                (const void *)k.data(), k.size()+1,
+                (const void *)v.data(), v.size()+1);
+        if(ret != 0) {
+            fprintf(stderr, "Error: sdskv_put() failed (iteration %d)\n", i);
+            sdskv_shutdown_service(kvcl, svr_addr);
+            sdskv_provider_handle_release(kvph);
+            margo_addr_free(mid, svr_addr);
+            sdskv_client_finalize(kvcl);
+            margo_finalize(mid);
+            return -1;
+        }
+        keys.push_back(k);
+        std::cerr << k << "\t ===> " << v << std::endl;
+    }
+    printf("Successfuly inserted %d keys\n", num_keys);
+
+    /* **** list keys **** */
+    std::sort(keys.begin(), keys.end());
+
+    auto i1 = keys.size()/4;
+    auto i2 = 3*keys.size()/4;
+    auto keys_after = keys[i1-1];
+
+    hg_size_t max_keys = i2-i1;
+    std::vector<std::vector<char>> result_strings(max_keys, std::vector<char>(max_key_size+1));
+    std::vector<void*> list_result(max_keys);
+    std::vector<hg_size_t> ksizes(max_keys, max_key_size+1);
+
+    for(unsigned i=0; i<max_keys; i++) {
+        list_result[i] = (void*)result_strings[i].data();
+    }
+
+    std::cout << "Expecting " << max_keys << " keys after " << keys_after << std::endl;
+
+    ret = sdskv_list_keys_with_prefix(kvph, db_id, 
+                (const void*)keys_after.c_str(), keys_after.size()+1,
+                prefix.data(), prefix.size(),
+                list_result.data(), ksizes.data(), &max_keys);
+
+    if(ret != 0) {
+        fprintf(stderr, "Error: sdskv_list_keys() failed\n");
+        sdskv_shutdown_service(kvcl, svr_addr);
+        sdskv_provider_handle_release(kvph);
+        margo_addr_free(mid, svr_addr);
+        sdskv_client_finalize(kvcl);
+        margo_finalize(mid);
+        return -1;
+    }
+
+    /* put the returned strings in an array */
+    std::vector<std::string> res;
+    for(auto ptr : list_result) {
+        res.push_back(std::string((const char*)ptr));
+        std::cout << *res.rbegin() << std::endl;
+    }
+
+    /* check that the returned keys are correct */
+    for(unsigned i=0; i < max_keys; i++) {
+        fprintf(stderr, "Returned key %d is %s\n", i, res[i].c_str());
+        if(strncmp(res[i].c_str(), prefix.c_str(), prefix.size()) != 0) {
+            fprintf(stderr, "Error: returned key doesn't match prefix\n");
+            fprintf(stderr, "       key received: %s\n", res[i].c_str());
+            fprintf(stderr, "       expected prefix: %s\n", prefix.c_str());
+            sdskv_shutdown_service(kvcl, svr_addr);
+            sdskv_provider_handle_release(kvph);
+            margo_addr_free(mid, svr_addr);
+            sdskv_client_finalize(kvcl);
+            margo_finalize(mid);
+            return -1;
+        }
+    }
+
+    /* shutdown the server */
+    ret = sdskv_shutdown_service(kvcl, svr_addr);
+
+    /**** cleanup ****/
+    sdskv_provider_handle_release(kvph);
+    margo_addr_free(mid, svr_addr);
+    sdskv_client_finalize(kvcl);
+    margo_finalize(mid);
+    return(ret);
+}
+
+static std::string gen_random_string(size_t len) {
+    static const char alphanum[] =
+                "0123456789"
+                "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                "abcdefghijklmnopqrstuvwxyz";
+    std::string s(len, ' ');
+    for (unsigned i = 0; i < len; ++i) {
+        s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];
+    }
+    return s;
+}