Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
sds
sds-keyval
Commits
6b64e158
Commit
6b64e158
authored
Feb 02, 2018
by
Matthieu Dorier
Browse files
started adding prefix feature
parent
fbb7e414
Changes
11
Hide whitespace changes
Inline
Side-by-side
src/datastore/berkeleydb_datastore.cc
View file @
6b64e158
...
...
@@ -214,7 +214,7 @@ void BerkeleyDBDataStore::set_in_memory(bool enable) {
_in_memory
=
enable
;
};
std
::
vector
<
ds_bulk_t
>
BerkeleyDBDataStore
::
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
)
std
::
vector
<
ds_bulk_t
>
BerkeleyDBDataStore
::
v
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
ds_bulk_t
>
keys
;
Dbc
*
cursorp
;
...
...
@@ -250,7 +250,7 @@ std::vector<ds_bulk_t> BerkeleyDBDataStore::list_keys(const ds_bulk_t &start, si
return
keys
;
}
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
BerkeleyDBDataStore
::
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
)
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
BerkeleyDBDataStore
::
v
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
keyvals
;
Dbc
*
cursorp
;
...
...
src/datastore/berkeleydb_datastore.h
View file @
6b64e158
...
...
@@ -20,9 +20,9 @@ public:
virtual
bool
get
(
const
ds_bulk_t
&
key
,
std
::
vector
<
ds_bulk_t
>
&
data
);
virtual
bool
erase
(
const
ds_bulk_t
&
key
);
virtual
void
set_in_memory
(
bool
enable
);
// enable/disable in-memory mode
virtual
std
::
vector
<
ds_bulk_t
>
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
);
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
);
protected:
virtual
std
::
vector
<
ds_bulk_t
>
vlist_keys
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
);
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
vlist_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
);
DbEnv
*
_dbenv
=
NULL
;
Db
*
_dbm
=
NULL
;
};
...
...
src/datastore/bwtree_datastore.cc
View file @
6b64e158
...
...
@@ -123,7 +123,7 @@ bool BwTreeDataStore::get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data) {
void
BwTreeDataStore
::
set_in_memory
(
bool
enable
)
{};
std
::
vector
<
ds_bulk_t
>
BwTreeDataStore
::
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
)
std
::
vector
<
ds_bulk_t
>
BwTreeDataStore
::
v
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
ds_bulk_t
>
keys
;
#if 0
...
...
@@ -136,7 +136,7 @@ std::vector<ds_bulk_t> BwTreeDataStore::list_keys(const ds_bulk_t &start, size_t
return
keys
;
}
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
BwTreeDataStore
::
list_keyvals
(
const
ds_bulk_t
&
start
,
size_t
count
)
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
BwTreeDataStore
::
v
list_keyvals
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
keyvals
;
#if 0
...
...
src/datastore/bwtree_datastore.h
View file @
6b64e158
...
...
@@ -20,9 +20,9 @@ public:
virtual
bool
get
(
const
ds_bulk_t
&
key
,
std
::
vector
<
ds_bulk_t
>
&
data
);
virtual
bool
erase
(
const
ds_bulk_t
&
key
);
virtual
void
set_in_memory
(
bool
enable
);
// a no-op
virtual
std
::
vector
<
ds_bulk_t
>
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
);
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
);
protected:
virtual
std
::
vector
<
ds_bulk_t
>
vlist_keys
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
);
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
vlist_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
);
BwTree
<
ds_bulk_t
,
ds_bulk_t
,
ds_bulk_less
,
ds_bulk_equal
,
ds_bulk_hash
,
ds_bulk_equal
,
ds_bulk_hash
>
*
_tree
=
NULL
;
...
...
src/datastore/datastore.h
View file @
6b64e158
...
...
@@ -22,13 +22,27 @@ public:
virtual
bool
get
(
const
ds_bulk_t
&
key
,
std
::
vector
<
ds_bulk_t
>
&
data
)
=
0
;
virtual
bool
erase
(
const
ds_bulk_t
&
key
)
=
0
;
virtual
void
set_in_memory
(
bool
enable
)
=
0
;
// enable/disable in-memory mode (where supported)
virtual
std
::
vector
<
ds_bulk_t
>
list_keys
(
const
ds_bulk_t
&
start_key
,
size_t
count
)
=
0
;
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
)
=
0
;
std
::
vector
<
ds_bulk_t
>
list_keys
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
=
ds_bulk_t
())
{
return
vlist_keys
(
start_key
,
count
,
prefix
);
}
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
=
ds_bulk_t
())
{
return
vlist_keyvals
(
start_key
,
count
,
prefix
);
}
protected:
Duplicates
_duplicates
;
bool
_eraseOnGet
;
bool
_debug
;
bool
_in_memory
;
virtual
std
::
vector
<
ds_bulk_t
>
vlist_keys
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
=
0
;
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
vlist_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
=
0
;
};
#endif // datastore_h
src/datastore/leveldb_datastore.cc
View file @
6b64e158
...
...
@@ -127,7 +127,7 @@ bool LevelDBDataStore::get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data) {
void
LevelDBDataStore
::
set_in_memory
(
bool
enable
)
{};
std
::
vector
<
ds_bulk_t
>
LevelDBDataStore
::
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
)
std
::
vector
<
ds_bulk_t
>
LevelDBDataStore
::
v
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
ds_bulk_t
>
keys
;
...
...
@@ -155,7 +155,7 @@ std::vector<ds_bulk_t> LevelDBDataStore::list_keys(const ds_bulk_t &start, size_
return
keys
;
}
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
LevelDBDataStore
::
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
)
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
LevelDBDataStore
::
v
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
keyvals
;
...
...
src/datastore/leveldb_datastore.h
View file @
6b64e158
...
...
@@ -20,9 +20,9 @@ public:
virtual
bool
get
(
const
ds_bulk_t
&
key
,
std
::
vector
<
ds_bulk_t
>
&
data
);
virtual
bool
erase
(
const
ds_bulk_t
&
key
);
virtual
void
set_in_memory
(
bool
enable
);
// not supported, a no-op
virtual
std
::
vector
<
ds_bulk_t
>
list_keys
(
const
ds_bulk_t
&
start
,
size_t
count
);
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
);
protected:
virtual
std
::
vector
<
ds_bulk_t
>
vlist_keys
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
);
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
vlist_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
);
leveldb
::
DB
*
_dbm
=
NULL
;
private:
std
::
string
toString
(
const
ds_bulk_t
&
key
);
...
...
src/datastore/map_datastore.h
View file @
6b64e158
...
...
@@ -55,7 +55,10 @@ class MapDataStore : public AbstractDataStore {
_in_memory
=
enable
;
}
virtual
std
::
vector
<
ds_bulk_t
>
list_keys
(
const
ds_bulk_t
&
start_key
,
size_t
count
)
{
protected:
virtual
std
::
vector
<
ds_bulk_t
>
vlist_keys
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
ds_bulk_t
>
result
;
auto
it
=
_map
.
lower_bound
(
start_key
);
while
(
it
!=
_map
.
end
()
&&
it
->
first
==
start_key
)
{
...
...
@@ -67,14 +70,26 @@ class MapDataStore : public AbstractDataStore {
return
result
;
}
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
list_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
)
{
virtual
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
vlist_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
result
;
auto
it
=
_map
.
lower_bound
(
start_key
);
while
(
it
!=
_map
.
end
()
&&
it
->
first
==
start_key
)
{
it
++
;
decltype
(
_map
.
begin
())
it
;
if
(
start_key
.
size
()
>
0
)
{
it
=
_map
.
lower_bound
(
start_key
);
while
(
it
!=
_map
.
end
()
&&
it
->
first
==
start_key
)
it
++
;
}
else
{
it
=
_map
.
begin
();
}
for
(
size_t
i
=
0
;
it
!=
_map
.
end
()
&&
i
<
count
;
it
++
,
i
++
)
{
result
.
push_back
(
*
it
);
while
(
result
.
size
()
<
count
&&
it
!=
_map
.
end
())
{
const
auto
&
p
=
*
it
;
if
(
prefix
.
size
()
>
p
.
first
.
size
())
continue
;
int
c
=
std
::
memcmp
(
prefix
.
data
(),
p
.
first
.
data
(),
prefix
.
size
());
if
(
c
==
0
)
{
result
.
push_back
(
*
it
);
}
else
if
(
c
<
0
)
{
break
;
// we have exceeded prefix
}
it
++
;
}
return
result
;
}
...
...
src/sdskv-client.c
View file @
6b64e158
...
...
@@ -541,11 +541,33 @@ int sdskv_list_keys(sdskv_provider_handle_t provider,
// representing sizes allocated in
// keys for each key
hg_size_t
*
max_keys
)
// maximum number of keys requested
{
return
sdskv_list_keys_with_prefix
(
provider
,
db_id
,
start_key
,
start_ksize
,
NULL
,
0
,
keys
,
ksizes
,
max_keys
);
}
int
sdskv_list_keys_with_prefix
(
sdskv_provider_handle_t
provider
,
sdskv_database_id_t
db_id
,
// db instance
const
void
*
start_key
,
// we want keys strictly after this start_key
hg_size_t
start_ksize
,
// size of the start_key
const
void
*
prefix
,
hg_size_t
prefix_size
,
void
**
keys
,
// pointer to an array of void* pointers,
// this array has size *max_keys
hg_size_t
*
ksizes
,
// pointer to an array of hg_size_t sizes
// representing sizes allocated in
// keys for each key
hg_size_t
*
max_keys
)
// maximum number of keys requested
{
list_keys_in_t
in
;
list_keys_out_t
out
;
in
.
keys_bulk_handle
=
HG_BULK_NULL
;
in
.
ksizes_bulk_handle
=
HG_BULK_NULL
;
hg_return_t
hret
=
HG_SUCCESS
;
hg_handle_t
handle
=
HG_HANDLE_NULL
;
int
ret
=
0
;
...
...
@@ -554,6 +576,10 @@ int sdskv_list_keys(sdskv_provider_handle_t provider,
in
.
db_id
=
db_id
;
in
.
start_key
.
data
=
(
kv_ptr_t
)
start_key
;
in
.
start_key
.
size
=
start_ksize
;
in
.
prefix
.
data
=
(
char
*
)
prefix
;
in
.
prefix
.
size
=
prefix_size
;
in
.
keys_bulk_handle
=
HG_BULK_NULL
;
in
.
ksizes_bulk_handle
=
HG_BULK_NULL
;
in
.
max_keys
=
*
max_keys
;
/* create bulk handle to expose the segments with key sizes */
...
...
@@ -640,13 +666,41 @@ int sdskv_list_keyvals(sdskv_provider_handle_t provider,
// representing sizes allocated in
// values for each value
hg_size_t
*
max_keys
)
// maximum number of keys requested
{
return
sdskv_list_keyvals_with_prefix
(
provider
,
db_id
,
start_key
,
start_ksize
,
NULL
,
0
,
keys
,
ksizes
,
values
,
vsizes
,
max_keys
);
}
int
sdskv_list_keyvals_with_prefix
(
sdskv_provider_handle_t
provider
,
sdskv_database_id_t
db_id
,
// db instance
const
void
*
start_key
,
// we want keys strictly after this start_key
hg_size_t
start_ksize
,
// size of the start_key
const
void
*
prefix
,
// prefix of returned keys
hg_size_t
prefix_size
,
// size of prefix
void
**
keys
,
// pointer to an array of void* pointers,
// this array has size *max_keys
hg_size_t
*
ksizes
,
// pointer to an array of hg_size_t sizes
// representing sizes allocated in
// keys for each key
void
**
values
,
// pointer to an array of void* pointers,
// this array has size *max_keys
hg_size_t
*
vsizes
,
// pointer to an array of hg_size_t sizes
// representing sizes allocated in
// values for each value
hg_size_t
*
max_keys
)
// maximum number of keys requested
{
list_keyvals_in_t
in
;
list_keyvals_out_t
out
;
in
.
keys_bulk_handle
=
HG_BULK_NULL
;
in
.
ksizes_bulk_handle
=
HG_BULK_NULL
;
in
.
vals_bulk_handle
=
HG_BULK_NULL
;
in
.
vsizes_bulk_handle
=
HG_BULK_NULL
;
hg_return_t
hret
=
HG_SUCCESS
;
hg_handle_t
handle
=
HG_HANDLE_NULL
;
int
ret
=
0
;
...
...
@@ -655,7 +709,13 @@ int sdskv_list_keyvals(sdskv_provider_handle_t provider,
in
.
db_id
=
db_id
;
in
.
start_key
.
data
=
(
kv_ptr_t
)
start_key
;
in
.
start_key
.
size
=
start_ksize
;
in
.
prefix
.
data
=
(
char
*
)
prefix
;
in
.
prefix
.
size
=
prefix_size
;
in
.
max_keys
=
*
max_keys
;
in
.
keys_bulk_handle
=
HG_BULK_NULL
;
in
.
ksizes_bulk_handle
=
HG_BULK_NULL
;
in
.
vals_bulk_handle
=
HG_BULK_NULL
;
in
.
vsizes_bulk_handle
=
HG_BULK_NULL
;
/* create bulk handle to expose the segments with key sizes */
hg_size_t
ksize_bulk_size
=
(
*
max_keys
)
*
sizeof
(
*
ksizes
);
...
...
src/sdskv-rpc-types.h
View file @
6b64e158
...
...
@@ -78,13 +78,16 @@ MERCURY_GEN_PROC(erase_in_t, ((uint64_t)(db_id))((kv_data_t)(key)))
// ------------- LIST KEYS ------------- //
MERCURY_GEN_PROC
(
list_keys_in_t
,
((
uint64_t
)(
db_id
))
\
((
kv_data_t
)(
start_key
))
\
((
kv_data_t
)(
prefix
))
\
((
hg_size_t
)(
max_keys
))
\
((
hg_bulk_t
)(
ksizes_bulk_handle
))
\
((
hg_bulk_t
)(
keys_bulk_handle
)))
MERCURY_GEN_PROC
(
list_keys_out_t
,
((
hg_size_t
)(
nkeys
))
((
int32_t
)(
ret
)))
// ------------- LIST KEYVALS ------------- //
MERCURY_GEN_PROC
(
list_keyvals_in_t
,
((
uint64_t
)(
db_id
))
\
((
kv_data_t
)(
start_key
))
\
((
kv_data_t
)(
prefix
))
\
((
hg_size_t
)(
max_keys
))
\
((
hg_bulk_t
)(
ksizes_bulk_handle
))
\
((
hg_bulk_t
)(
keys_bulk_handle
))
\
...
...
src/sdskv-server.cc
View file @
6b64e158
...
...
@@ -687,7 +687,8 @@ static void sdskv_list_keys_ult(hg_handle_t handle)
/* get the keys from the underlying database */
ds_bulk_t
start_kdata
(
in
.
start_key
.
data
,
in
.
start_key
.
data
+
in
.
start_key
.
size
);
auto
keys
=
db
->
list_keys
(
start_kdata
,
in
.
max_keys
);
ds_bulk_t
prefix
(
in
.
prefix
.
data
,
in
.
prefix
.
data
+
in
.
prefix
.
size
);
auto
keys
=
db
->
list_keys
(
start_kdata
,
in
.
max_keys
,
prefix
);
hg_size_t
num_keys
=
std
::
min
(
keys
.
size
(),
in
.
max_keys
);
/* create the array of actual sizes */
...
...
@@ -855,7 +856,8 @@ static void sdskv_list_keyvals_ult(hg_handle_t handle)
/* get the keys and values from the underlying database */
ds_bulk_t
start_kdata
(
in
.
start_key
.
data
,
in
.
start_key
.
data
+
in
.
start_key
.
size
);
auto
keyvals
=
db
->
list_keyvals
(
start_kdata
,
in
.
max_keys
);
ds_bulk_t
prefix
(
in
.
prefix
.
data
,
in
.
prefix
.
data
+
in
.
prefix
.
size
);
auto
keyvals
=
db
->
list_keyvals
(
start_kdata
,
in
.
max_keys
,
prefix
);
hg_size_t
num_keys
=
std
::
min
(
keyvals
.
size
(),
in
.
max_keys
);
/* create the array of actual key sizes */
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment