started adding prefix feature

src/datastore/berkeleydb_datastore.cc

@@ -214,7 +214,7 @@ void BerkeleyDBDataStore::set_in_memory(bool enable) {
   _in_memory = enable;
 };
 
-std::vector<ds_bulk_t> BerkeleyDBDataStore::list_keys(const ds_bulk_t &start, size_t count)
+std::vector<ds_bulk_t> BerkeleyDBDataStore::vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
 {
     std::vector<ds_bulk_t> keys;
     Dbc * cursorp;
@@ -250,7 +250,7 @@ std::vector<ds_bulk_t> BerkeleyDBDataStore::list_keys(const ds_bulk_t &start, si
     return keys;
 }
 
-std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::list_keyvals(const ds_bulk_t &start_key, size_t count)
+std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
 {
     std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
     Dbc * cursorp;

src/datastore/berkeleydb_datastore.h

@@ -20,9 +20,9 @@ public:
   virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data);
   virtual bool erase(const ds_bulk_t &key);
   virtual void set_in_memory(bool enable); // enable/disable in-memory mode
-  virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start, size_t count);
-  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count);
 protected:
+  virtual std::vector<ds_bulk_t> vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix);
+  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &);
   DbEnv *_dbenv = NULL;
   Db *_dbm = NULL;
 };

src/datastore/bwtree_datastore.cc

@@ -123,7 +123,7 @@ bool BwTreeDataStore::get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data) {
 void BwTreeDataStore::set_in_memory(bool enable)
 {};
 
-std::vector<ds_bulk_t> BwTreeDataStore::list_keys(const ds_bulk_t &start, size_t count)
+std::vector<ds_bulk_t> BwTreeDataStore::vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
 {
     std::vector<ds_bulk_t> keys;
 #if 0
@@ -136,7 +136,7 @@ std::vector<ds_bulk_t> BwTreeDataStore::list_keys(const ds_bulk_t &start, size_t
     return keys;
 }
 
-std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BwTreeDataStore::list_keyvals(const ds_bulk_t &start, size_t count)
+std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BwTreeDataStore::vlist_keyvals(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
 {
     std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
 #if 0

src/datastore/bwtree_datastore.h

@@ -20,9 +20,9 @@ public:
   virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data);
   virtual bool erase(const ds_bulk_t &key);
   virtual void set_in_memory(bool enable); // a no-op
-  virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start, size_t count);
-  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count);
 protected:
+  virtual std::vector<ds_bulk_t> vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix);
+  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix);
   BwTree<ds_bulk_t, ds_bulk_t, 
 	 ds_bulk_less, ds_bulk_equal, ds_bulk_hash,
 	 ds_bulk_equal, ds_bulk_hash> *_tree = NULL;

src/datastore/datastore.h

@@ -22,13 +22,27 @@ public:
   virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data)=0;
   virtual bool erase(const ds_bulk_t &key) = 0;
   virtual void set_in_memory(bool enable)=0; // enable/disable in-memory mode (where supported)
-  virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start_key, size_t count)=0;
-  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count)=0;
+
+  std::vector<ds_bulk_t> list_keys(
+          const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix=ds_bulk_t()) {
+      return vlist_keys(start_key, count, prefix);
+  }
+
+  std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(
+          const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix=ds_bulk_t()) {
+    return vlist_keyvals(start_key, count, prefix);
+  }
+
 protected:
   Duplicates _duplicates;
   bool _eraseOnGet;
   bool _debug;
   bool _in_memory;
+
+  virtual std::vector<ds_bulk_t> vlist_keys(
+          const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix) = 0;
+  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(
+          const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix) = 0;
 };
 
 #endif // datastore_h

src/datastore/leveldb_datastore.cc

@@ -127,7 +127,7 @@ bool LevelDBDataStore::get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data) {
 void LevelDBDataStore::set_in_memory(bool enable)
 {};
 
-std::vector<ds_bulk_t> LevelDBDataStore::list_keys(const ds_bulk_t &start, size_t count)
+std::vector<ds_bulk_t> LevelDBDataStore::vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
 {
     std::vector<ds_bulk_t> keys;
 
@@ -155,7 +155,7 @@ std::vector<ds_bulk_t> LevelDBDataStore::list_keys(const ds_bulk_t &start, size_
     return keys;
 }
 
-std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::list_keyvals(const ds_bulk_t &start_key, size_t count)
+std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
 {
     std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
 

src/datastore/leveldb_datastore.h

@@ -20,9 +20,9 @@ public:
   virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data);
   virtual bool erase(const ds_bulk_t &key);
   virtual void set_in_memory(bool enable); // not supported, a no-op
-  virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start, size_t count);
-  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count);
 protected:
+  virtual std::vector<ds_bulk_t> vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix);
+  virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix);
   leveldb::DB *_dbm = NULL;
 private:
   std::string toString(const ds_bulk_t &key);

src/datastore/map_datastore.h

@@ -55,7 +55,10 @@ class MapDataStore : public AbstractDataStore {
             _in_memory = enable;
         }
 
-        virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start_key, size_t count) {
+    protected:
+
+        virtual std::vector<ds_bulk_t> vlist_keys(
+                const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix) {
             std::vector<ds_bulk_t> result;
             auto it = _map.lower_bound(start_key);
             while(it != _map.end() && it->first == start_key) {
@@ -67,14 +70,26 @@ class MapDataStore : public AbstractDataStore {
             return result;
         }
 
-        virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count) {
+        virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(
+                const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix) {
             std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;
-            auto it = _map.lower_bound(start_key);
-            while(it != _map.end() && it->first == start_key) {
-                it++;
+            decltype(_map.begin()) it;
+            if(start_key.size() > 0) {
+                it = _map.lower_bound(start_key);
+                while(it != _map.end() && it->first == start_key) it++;
+            } else {
+                it = _map.begin();
             }
-            for(size_t i=0; it != _map.end() && i < count; it++, i++) {
-                result.push_back(*it);
+            while(result.size() < count && it != _map.end()) {
+                const auto& p = *it;
+                if(prefix.size() > p.first.size()) continue;
+                int c = std::memcmp(prefix.data(), p.first.data(), prefix.size());
+                if(c == 0) {
+                    result.push_back(*it);
+                } else if(c < 0) {
+                    break; // we have exceeded prefix
+                }
+                it++;
             }
             return result;
         }

src/sdskv-client.c

@@ -541,11 +541,33 @@ int sdskv_list_keys(sdskv_provider_handle_t provider,
         //    representing sizes allocated in
         //     keys for each key
         hg_size_t* max_keys)   // maximum number of keys requested
+{
+    return sdskv_list_keys_with_prefix(provider,
+            db_id,
+            start_key,
+            start_ksize,
+            NULL,
+            0,
+            keys,
+            ksizes,
+            max_keys);
+}
+
+int sdskv_list_keys_with_prefix(sdskv_provider_handle_t provider,
+        sdskv_database_id_t db_id,  // db instance
+        const void *start_key,  // we want keys strictly after this start_key
+        hg_size_t start_ksize,  // size of the start_key
+        const void *prefix,
+        hg_size_t prefix_size,
+        void **keys,            // pointer to an array of void* pointers,
+        //     this array has size *max_keys
+        hg_size_t* ksizes,      // pointer to an array of hg_size_t sizes
+        //    representing sizes allocated in
+        //     keys for each key
+        hg_size_t* max_keys)   // maximum number of keys requested
 {
     list_keys_in_t  in;
     list_keys_out_t out;
-    in.keys_bulk_handle   = HG_BULK_NULL;
-    in.ksizes_bulk_handle = HG_BULK_NULL;
     hg_return_t hret      = HG_SUCCESS;
     hg_handle_t handle    = HG_HANDLE_NULL;
     int ret = 0;
@@ -554,6 +576,10 @@ int sdskv_list_keys(sdskv_provider_handle_t provider,
     in.db_id = db_id;
     in.start_key.data = (kv_ptr_t) start_key;
     in.start_key.size = start_ksize;
+    in.prefix.data = (char*)prefix;
+    in.prefix.size = prefix_size;
+    in.keys_bulk_handle   = HG_BULK_NULL;
+    in.ksizes_bulk_handle = HG_BULK_NULL;
     in.max_keys = *max_keys;
 
     /* create bulk handle to expose the segments with key sizes */
@@ -640,13 +666,41 @@ int sdskv_list_keyvals(sdskv_provider_handle_t provider,
                                 //    representing sizes allocated in
                                 //    values for each value
         hg_size_t* max_keys)    // maximum number of keys requested
+{
+    return sdskv_list_keyvals_with_prefix(
+            provider,
+            db_id,
+            start_key,
+            start_ksize,
+            NULL,
+            0,
+            keys,
+            ksizes,
+            values,
+            vsizes,
+            max_keys);
+}
+
+int sdskv_list_keyvals_with_prefix(sdskv_provider_handle_t provider,
+        sdskv_database_id_t db_id,  // db instance
+        const void *start_key,  // we want keys strictly after this start_key
+        hg_size_t start_ksize,  // size of the start_key
+        const void* prefix,     // prefix of returned keys
+        hg_size_t prefix_size,  // size of prefix
+        void **keys,            // pointer to an array of void* pointers,
+                                //     this array has size *max_keys
+        hg_size_t* ksizes,      // pointer to an array of hg_size_t sizes
+                                //    representing sizes allocated in
+                                //     keys for each key
+        void **values,          // pointer to an array of void* pointers,
+                                //     this array has size *max_keys
+        hg_size_t* vsizes,      // pointer to an array of hg_size_t sizes
+                                //    representing sizes allocated in
+                                //    values for each value
+        hg_size_t* max_keys)    // maximum number of keys requested
 {
     list_keyvals_in_t  in;
     list_keyvals_out_t out;
-    in.keys_bulk_handle   = HG_BULK_NULL;
-    in.ksizes_bulk_handle = HG_BULK_NULL;
-    in.vals_bulk_handle   = HG_BULK_NULL;
-    in.vsizes_bulk_handle = HG_BULK_NULL;
     hg_return_t hret      = HG_SUCCESS;
     hg_handle_t handle    = HG_HANDLE_NULL;
     int ret = 0;
@@ -655,7 +709,13 @@ int sdskv_list_keyvals(sdskv_provider_handle_t provider,
     in.db_id = db_id;
     in.start_key.data = (kv_ptr_t) start_key;
     in.start_key.size = start_ksize;
+    in.prefix.data = (char*)prefix;
+    in.prefix.size = prefix_size;
     in.max_keys = *max_keys;
+    in.keys_bulk_handle   = HG_BULK_NULL;
+    in.ksizes_bulk_handle = HG_BULK_NULL;
+    in.vals_bulk_handle   = HG_BULK_NULL;
+    in.vsizes_bulk_handle = HG_BULK_NULL;
 
     /* create bulk handle to expose the segments with key sizes */
     hg_size_t ksize_bulk_size = (*max_keys)*sizeof(*ksizes);

src/sdskv-rpc-types.h

@@ -78,13 +78,16 @@ MERCURY_GEN_PROC(erase_in_t, ((uint64_t)(db_id))((kv_data_t)(key)))
 // ------------- LIST KEYS ------------- //
 MERCURY_GEN_PROC(list_keys_in_t, ((uint64_t)(db_id))\
         ((kv_data_t)(start_key))\
+        ((kv_data_t)(prefix))\
         ((hg_size_t)(max_keys))\
         ((hg_bulk_t)(ksizes_bulk_handle))\
         ((hg_bulk_t)(keys_bulk_handle)))
 MERCURY_GEN_PROC(list_keys_out_t, ((hg_size_t)(nkeys)) ((int32_t)(ret)))
+
 // ------------- LIST KEYVALS ------------- //
 MERCURY_GEN_PROC(list_keyvals_in_t, ((uint64_t)(db_id))\
         ((kv_data_t)(start_key))\
+        ((kv_data_t)(prefix))\
         ((hg_size_t)(max_keys))\
         ((hg_bulk_t)(ksizes_bulk_handle))\
         ((hg_bulk_t)(keys_bulk_handle))\

src/sdskv-server.cc

@@ -687,7 +687,8 @@ static void sdskv_list_keys_ult(hg_handle_t handle)
 
         /* get the keys from the underlying database */    
         ds_bulk_t start_kdata(in.start_key.data, in.start_key.data+in.start_key.size);
-        auto keys = db->list_keys(start_kdata, in.max_keys);
+        ds_bulk_t prefix(in.prefix.data, in.prefix.data+in.prefix.size);
+        auto keys = db->list_keys(start_kdata, in.max_keys, prefix);
         hg_size_t num_keys = std::min(keys.size(), in.max_keys);
 
         /* create the array of actual sizes */
@@ -855,7 +856,8 @@ static void sdskv_list_keyvals_ult(hg_handle_t handle)
 
         /* get the keys and values from the underlying database */    
         ds_bulk_t start_kdata(in.start_key.data, in.start_key.data+in.start_key.size);
-        auto keyvals = db->list_keyvals(start_kdata, in.max_keys);
+        ds_bulk_t prefix(in.prefix.data, in.prefix.data+in.prefix.size);
+        auto keyvals = db->list_keyvals(start_kdata, in.max_keys, prefix);
         hg_size_t num_keys = std::min(keyvals.size(), in.max_keys);
 
         /* create the array of actual key sizes */