Commit 6b64e158 authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

started adding prefix feature

parent fbb7e414
......@@ -214,7 +214,7 @@ void BerkeleyDBDataStore::set_in_memory(bool enable) {
_in_memory = enable;
};
std::vector<ds_bulk_t> BerkeleyDBDataStore::list_keys(const ds_bulk_t &start, size_t count)
std::vector<ds_bulk_t> BerkeleyDBDataStore::vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
{
std::vector<ds_bulk_t> keys;
Dbc * cursorp;
......@@ -250,7 +250,7 @@ std::vector<ds_bulk_t> BerkeleyDBDataStore::list_keys(const ds_bulk_t &start, si
return keys;
}
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::list_keyvals(const ds_bulk_t &start_key, size_t count)
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BerkeleyDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
{
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
Dbc * cursorp;
......
......@@ -20,9 +20,9 @@ public:
virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data);
virtual bool erase(const ds_bulk_t &key);
virtual void set_in_memory(bool enable); // enable/disable in-memory mode
virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start, size_t count);
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count);
protected:
virtual std::vector<ds_bulk_t> vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix);
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &);
DbEnv *_dbenv = NULL;
Db *_dbm = NULL;
};
......
......@@ -123,7 +123,7 @@ bool BwTreeDataStore::get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data) {
void BwTreeDataStore::set_in_memory(bool enable)
{};
std::vector<ds_bulk_t> BwTreeDataStore::list_keys(const ds_bulk_t &start, size_t count)
std::vector<ds_bulk_t> BwTreeDataStore::vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
{
std::vector<ds_bulk_t> keys;
#if 0
......@@ -136,7 +136,7 @@ std::vector<ds_bulk_t> BwTreeDataStore::list_keys(const ds_bulk_t &start, size_t
return keys;
}
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BwTreeDataStore::list_keyvals(const ds_bulk_t &start, size_t count)
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> BwTreeDataStore::vlist_keyvals(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
{
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
#if 0
......
......@@ -20,9 +20,9 @@ public:
virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data);
virtual bool erase(const ds_bulk_t &key);
virtual void set_in_memory(bool enable); // a no-op
virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start, size_t count);
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count);
protected:
virtual std::vector<ds_bulk_t> vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix);
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix);
BwTree<ds_bulk_t, ds_bulk_t,
ds_bulk_less, ds_bulk_equal, ds_bulk_hash,
ds_bulk_equal, ds_bulk_hash> *_tree = NULL;
......
......@@ -22,13 +22,27 @@ public:
virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data)=0;
virtual bool erase(const ds_bulk_t &key) = 0;
virtual void set_in_memory(bool enable)=0; // enable/disable in-memory mode (where supported)
virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start_key, size_t count)=0;
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count)=0;
std::vector<ds_bulk_t> list_keys(
const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix=ds_bulk_t()) {
return vlist_keys(start_key, count, prefix);
}
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(
const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix=ds_bulk_t()) {
return vlist_keyvals(start_key, count, prefix);
}
protected:
Duplicates _duplicates;
bool _eraseOnGet;
bool _debug;
bool _in_memory;
virtual std::vector<ds_bulk_t> vlist_keys(
const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix) = 0;
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(
const ds_bulk_t &start_key, size_t count, const ds_bulk_t& prefix) = 0;
};
#endif // datastore_h
......@@ -127,7 +127,7 @@ bool LevelDBDataStore::get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data) {
void LevelDBDataStore::set_in_memory(bool enable)
{};
std::vector<ds_bulk_t> LevelDBDataStore::list_keys(const ds_bulk_t &start, size_t count)
std::vector<ds_bulk_t> LevelDBDataStore::vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix)
{
std::vector<ds_bulk_t> keys;
......@@ -155,7 +155,7 @@ std::vector<ds_bulk_t> LevelDBDataStore::list_keys(const ds_bulk_t &start, size_
return keys;
}
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::list_keyvals(const ds_bulk_t &start_key, size_t count)
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix)
{
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
......
......@@ -20,9 +20,9 @@ public:
virtual bool get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data);
virtual bool erase(const ds_bulk_t &key);
virtual void set_in_memory(bool enable); // not supported, a no-op
virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start, size_t count);
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count);
protected:
virtual std::vector<ds_bulk_t> vlist_keys(const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix);
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix);
leveldb::DB *_dbm = NULL;
private:
std::string toString(const ds_bulk_t &key);
......
......@@ -55,7 +55,10 @@ class MapDataStore : public AbstractDataStore {
_in_memory = enable;
}
virtual std::vector<ds_bulk_t> list_keys(const ds_bulk_t &start_key, size_t count) {
protected:
virtual std::vector<ds_bulk_t> vlist_keys(
const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix) {
std::vector<ds_bulk_t> result;
auto it = _map.lower_bound(start_key);
while(it != _map.end() && it->first == start_key) {
......@@ -67,14 +70,26 @@ class MapDataStore : public AbstractDataStore {
return result;
}
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> list_keyvals(const ds_bulk_t &start_key, size_t count) {
virtual std::vector<std::pair<ds_bulk_t,ds_bulk_t>> vlist_keyvals(
const ds_bulk_t &start_key, size_t count, const ds_bulk_t &prefix) {
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;
auto it = _map.lower_bound(start_key);
while(it != _map.end() && it->first == start_key) {
it++;
decltype(_map.begin()) it;
if(start_key.size() > 0) {
it = _map.lower_bound(start_key);
while(it != _map.end() && it->first == start_key) it++;
} else {
it = _map.begin();
}
for(size_t i=0; it != _map.end() && i < count; it++, i++) {
result.push_back(*it);
while(result.size() < count && it != _map.end()) {
const auto& p = *it;
if(prefix.size() > p.first.size()) continue;
int c = std::memcmp(prefix.data(), p.first.data(), prefix.size());
if(c == 0) {
result.push_back(*it);
} else if(c < 0) {
break; // we have exceeded prefix
}
it++;
}
return result;
}
......
......@@ -541,11 +541,33 @@ int sdskv_list_keys(sdskv_provider_handle_t provider,
// representing sizes allocated in
// keys for each key
hg_size_t* max_keys) // maximum number of keys requested
{
return sdskv_list_keys_with_prefix(provider,
db_id,
start_key,
start_ksize,
NULL,
0,
keys,
ksizes,
max_keys);
}
int sdskv_list_keys_with_prefix(sdskv_provider_handle_t provider,
sdskv_database_id_t db_id, // db instance
const void *start_key, // we want keys strictly after this start_key
hg_size_t start_ksize, // size of the start_key
const void *prefix,
hg_size_t prefix_size,
void **keys, // pointer to an array of void* pointers,
// this array has size *max_keys
hg_size_t* ksizes, // pointer to an array of hg_size_t sizes
// representing sizes allocated in
// keys for each key
hg_size_t* max_keys) // maximum number of keys requested
{
list_keys_in_t in;
list_keys_out_t out;
in.keys_bulk_handle = HG_BULK_NULL;
in.ksizes_bulk_handle = HG_BULK_NULL;
hg_return_t hret = HG_SUCCESS;
hg_handle_t handle = HG_HANDLE_NULL;
int ret = 0;
......@@ -554,6 +576,10 @@ int sdskv_list_keys(sdskv_provider_handle_t provider,
in.db_id = db_id;
in.start_key.data = (kv_ptr_t) start_key;
in.start_key.size = start_ksize;
in.prefix.data = (char*)prefix;
in.prefix.size = prefix_size;
in.keys_bulk_handle = HG_BULK_NULL;
in.ksizes_bulk_handle = HG_BULK_NULL;
in.max_keys = *max_keys;
/* create bulk handle to expose the segments with key sizes */
......@@ -640,13 +666,41 @@ int sdskv_list_keyvals(sdskv_provider_handle_t provider,
// representing sizes allocated in
// values for each value
hg_size_t* max_keys) // maximum number of keys requested
{
return sdskv_list_keyvals_with_prefix(
provider,
db_id,
start_key,
start_ksize,
NULL,
0,
keys,
ksizes,
values,
vsizes,
max_keys);
}
int sdskv_list_keyvals_with_prefix(sdskv_provider_handle_t provider,
sdskv_database_id_t db_id, // db instance
const void *start_key, // we want keys strictly after this start_key
hg_size_t start_ksize, // size of the start_key
const void* prefix, // prefix of returned keys
hg_size_t prefix_size, // size of prefix
void **keys, // pointer to an array of void* pointers,
// this array has size *max_keys
hg_size_t* ksizes, // pointer to an array of hg_size_t sizes
// representing sizes allocated in
// keys for each key
void **values, // pointer to an array of void* pointers,
// this array has size *max_keys
hg_size_t* vsizes, // pointer to an array of hg_size_t sizes
// representing sizes allocated in
// values for each value
hg_size_t* max_keys) // maximum number of keys requested
{
list_keyvals_in_t in;
list_keyvals_out_t out;
in.keys_bulk_handle = HG_BULK_NULL;
in.ksizes_bulk_handle = HG_BULK_NULL;
in.vals_bulk_handle = HG_BULK_NULL;
in.vsizes_bulk_handle = HG_BULK_NULL;
hg_return_t hret = HG_SUCCESS;
hg_handle_t handle = HG_HANDLE_NULL;
int ret = 0;
......@@ -655,7 +709,13 @@ int sdskv_list_keyvals(sdskv_provider_handle_t provider,
in.db_id = db_id;
in.start_key.data = (kv_ptr_t) start_key;
in.start_key.size = start_ksize;
in.prefix.data = (char*)prefix;
in.prefix.size = prefix_size;
in.max_keys = *max_keys;
in.keys_bulk_handle = HG_BULK_NULL;
in.ksizes_bulk_handle = HG_BULK_NULL;
in.vals_bulk_handle = HG_BULK_NULL;
in.vsizes_bulk_handle = HG_BULK_NULL;
/* create bulk handle to expose the segments with key sizes */
hg_size_t ksize_bulk_size = (*max_keys)*sizeof(*ksizes);
......
......@@ -78,13 +78,16 @@ MERCURY_GEN_PROC(erase_in_t, ((uint64_t)(db_id))((kv_data_t)(key)))
// ------------- LIST KEYS ------------- //
MERCURY_GEN_PROC(list_keys_in_t, ((uint64_t)(db_id))\
((kv_data_t)(start_key))\
((kv_data_t)(prefix))\
((hg_size_t)(max_keys))\
((hg_bulk_t)(ksizes_bulk_handle))\
((hg_bulk_t)(keys_bulk_handle)))
MERCURY_GEN_PROC(list_keys_out_t, ((hg_size_t)(nkeys)) ((int32_t)(ret)))
// ------------- LIST KEYVALS ------------- //
MERCURY_GEN_PROC(list_keyvals_in_t, ((uint64_t)(db_id))\
((kv_data_t)(start_key))\
((kv_data_t)(prefix))\
((hg_size_t)(max_keys))\
((hg_bulk_t)(ksizes_bulk_handle))\
((hg_bulk_t)(keys_bulk_handle))\
......
......@@ -687,7 +687,8 @@ static void sdskv_list_keys_ult(hg_handle_t handle)
/* get the keys from the underlying database */
ds_bulk_t start_kdata(in.start_key.data, in.start_key.data+in.start_key.size);
auto keys = db->list_keys(start_kdata, in.max_keys);
ds_bulk_t prefix(in.prefix.data, in.prefix.data+in.prefix.size);
auto keys = db->list_keys(start_kdata, in.max_keys, prefix);
hg_size_t num_keys = std::min(keys.size(), in.max_keys);
/* create the array of actual sizes */
......@@ -855,7 +856,8 @@ static void sdskv_list_keyvals_ult(hg_handle_t handle)
/* get the keys and values from the underlying database */
ds_bulk_t start_kdata(in.start_key.data, in.start_key.data+in.start_key.size);
auto keyvals = db->list_keyvals(start_kdata, in.max_keys);
ds_bulk_t prefix(in.prefix.data, in.prefix.data+in.prefix.size);
auto keyvals = db->list_keyvals(start_kdata, in.max_keys, prefix);
hg_size_t num_keys = std::min(keyvals.size(), in.max_keys);
/* create the array of actual key sizes */
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment