Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
sds
sds-keyval
Commits
5f8918aa
Commit
5f8918aa
authored
Feb 02, 2018
by
Matthieu Dorier
Browse files
added prefix feature to BDB and LDB, and tested
parent
30801a72
Changes
15
Hide whitespace changes
Inline
Side-by-side
configure.ac
View file @
5f8918aa
...
...
@@ -102,8 +102,8 @@ AC_ARG_ENABLE([leveldb],
)
AC_ARG_ENABLE([bwtree],
AS_HELP_STRING([--enable-bwtree], [Enable BwTree as server
backend. (default is
yes
) ]),
[bwtree_backend=${enableval}], [bwtree_backend=
yes
]
backend. (default is
no
) ]),
[bwtree_backend=${enableval}], [bwtree_backend=
no
]
)
...
...
@@ -129,6 +129,7 @@ fi
if test "x${bwtree_backend}" == xyes ; then
AC_DEFINE([USE_BWTREE], 1, [use BwTree backend])
AC_MSG_WARN([BwTree backend is deprecated])
CPPFLAGS="-I${srcdir}/src/BwTree/src ${CPPFLAGS}"
CXXFLAGS="-pthread -std=c++11 -g -Wall -mcx16 -Wno-invalid-offsetof ${CXXFLAGS}"
fi
...
...
src/datastore/berkeleydb_datastore.cc
View file @
5f8918aa
...
...
@@ -224,50 +224,94 @@ std::vector<ds_bulk_t> BerkeleyDBDataStore::vlist_keys(const ds_bulk_t &start, s
/* 'start' is like RADOS: not inclusive */
if
(
start
.
size
())
{
key
.
set_size
(
start
.
size
());
key
.
set_data
((
void
*
)
start
.
data
());
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_SET_RANGE
);
if
(
ret
!=
0
)
{
cursorp
->
close
();
return
keys
;
}
ds_bulk_t
k
(
key
.
get_size
()
);
memcpy
(
k
.
data
(),
key
.
get_data
(),
key
.
get_size
()
);
key
.
set_size
(
start
.
size
());
key
.
set_data
((
void
*
)
start
.
data
());
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_SET_RANGE
);
if
(
ret
!=
0
)
{
cursorp
->
close
();
return
keys
;
}
}
else
{
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_FIRST
);
if
(
ret
!=
0
)
{
cursorp
->
close
();
return
keys
;
}
}
ds_bulk_t
k
((
char
*
)
key
.
get_data
(),
((
char
*
)
key
.
get_data
())
+
key
.
get_size
());
/* SET_RANGE will return the smallest key greater than or equal to the
* requested key, but we want strictly greater than */
if
(
k
!=
start
)
keys
.
push_back
(
std
::
move
(
k
));
int
c
=
0
;
if
(
k
!=
start
)
{
c
=
std
::
memcmp
(
prefix
.
data
(),
k
.
data
(),
prefix
.
size
());
if
(
c
==
0
)
{
keys
.
push_back
(
std
::
move
(
k
));
}
}
while
(
keys
.
size
()
<
count
)
{
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_NEXT
);
if
(
ret
!=
0
)
break
;
ds_bulk_t
k
(
key
.
get_size
()
);
memcpy
(
k
.
data
(),
key
.
get_data
(),
key
.
get_size
()
);
keys
.
push_back
(
std
::
move
(
k
));
while
(
keys
.
size
()
<
count
&&
c
>=
0
)
{
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_NEXT
);
if
(
ret
!=
0
)
break
;
ds_bulk_t
k
((
char
*
)
key
.
get_data
(),
((
char
*
)
key
.
get_data
())
+
key
.
get_size
());
c
=
std
::
memcmp
(
prefix
.
data
(),
k
.
data
(),
prefix
.
size
());
if
(
c
==
0
)
{
keys
.
push_back
(
std
::
move
(
k
));
}
}
cursorp
->
close
();
return
keys
;
}
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
BerkeleyDBDataStore
::
vlist_keyvals
(
const
ds_bulk_t
&
start
_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
BerkeleyDBDataStore
::
vlist_keyvals
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
keyvals
;
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
result
;
Dbc
*
cursorp
;
Dbt
key
,
data
;
int
ret
;
_dbm
->
cursor
(
NULL
,
&
cursorp
,
0
);
for
(
size_t
i
=
0
;
i
<
count
;
i
++
)
{
int
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_NEXT
);
if
(
ret
!=
0
)
break
;
ds_bulk_t
k
(
key
.
get_size
());
ds_bulk_t
v
(
data
.
get_size
());
/* 'start' is like RADOS: not inclusive */
if
(
start
.
size
())
{
key
.
set_size
(
start
.
size
());
key
.
set_data
((
void
*
)
start
.
data
());
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_SET_RANGE
);
if
(
ret
!=
0
)
{
cursorp
->
close
();
return
result
;
}
}
else
{
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_FIRST
);
if
(
ret
!=
0
)
{
cursorp
->
close
();
return
result
;
}
}
ds_bulk_t
k
((
char
*
)
key
.
get_data
(),
((
char
*
)
key
.
get_data
())
+
key
.
get_size
());
ds_bulk_t
v
((
char
*
)
data
.
get_data
(),
((
char
*
)
data
.
get_data
())
+
data
.
get_size
());
memcpy
(
k
.
data
(),
key
.
get_data
(),
key
.
get_size
());
memcpy
(
v
.
data
(),
data
.
get_data
(),
data
.
get_size
());
/* SET_RANGE will return the smallest key greater than or equal to the
* requested key, but we want strictly greater than */
int
c
=
0
;
if
(
k
!=
start
)
{
c
=
std
::
memcmp
(
prefix
.
data
(),
k
.
data
(),
prefix
.
size
());
if
(
c
==
0
)
{
result
.
push_back
(
std
::
make_pair
(
std
::
move
(
k
),
std
::
move
(
v
)));
}
}
while
(
result
.
size
()
<
count
&&
c
>=
0
)
{
ret
=
cursorp
->
get
(
&
key
,
&
data
,
DB_NEXT
);
if
(
ret
!=
0
)
break
;
ds_bulk_t
k
((
char
*
)
key
.
get_data
(),
((
char
*
)
key
.
get_data
())
+
key
.
get_size
());
ds_bulk_t
v
((
char
*
)
data
.
get_data
(),
((
char
*
)
data
.
get_data
())
+
data
.
get_size
());
keyvals
.
push_back
(
std
::
make_pair
(
std
::
move
(
k
),
std
::
move
(
v
)));
c
=
std
::
memcmp
(
prefix
.
data
(),
k
.
data
(),
prefix
.
size
());
if
(
c
==
0
)
{
result
.
push_back
(
std
::
make_pair
(
std
::
move
(
k
),
std
::
move
(
v
)));
}
}
cursorp
->
close
();
return
keyvals
;
return
result
;
}
src/datastore/leveldb_datastore.cc
View file @
5f8918aa
...
...
@@ -134,28 +134,72 @@ std::vector<ds_bulk_t> LevelDBDataStore::vlist_keys(const ds_bulk_t &start, size
leveldb
::
Iterator
*
it
=
_dbm
->
NewIterator
(
leveldb
::
ReadOptions
());
leveldb
::
Slice
start_slice
(
start
.
data
(),
start
.
size
());
int
c
=
0
;
if
(
start
.
size
()
>
0
)
{
it
->
Seek
(
start_slice
);
/* we treat 'start' the way RADOS treats it: excluding it from returned
* keys. LevelDB treats start inclusively, so skip over it if we found
* an exact match */
if
(
start
.
size
()
==
it
->
key
().
size
()
&&
(
memcmp
(
it
->
key
().
data
(),
start
.
data
(),
start
.
size
())
==
0
))
it
->
Next
();
it
->
Seek
(
start_slice
);
/* we treat 'start' the way RADOS treats it: excluding it from returned
* keys. LevelDB treats start inclusively, so skip over it if we found
* an exact match */
if
(
start
.
size
()
==
it
->
key
().
size
()
&&
(
memcmp
(
it
->
key
().
data
(),
start
.
data
(),
start
.
size
())
==
0
))
it
->
Next
();
}
else
{
it
->
SeekToFirst
();
it
->
SeekToFirst
();
}
/* note: iterator initialized above, not in for loop */
for
(;
it
->
Valid
()
&&
keys
.
size
()
<
count
;
it
->
Next
()
)
{
ds_bulk_t
k
(
it
->
key
().
size
());
memcpy
(
k
.
data
(),
it
->
key
().
data
(),
it
->
key
().
size
()
);
keys
.
push_back
(
std
::
move
(
k
));
c
=
std
::
memcmp
(
prefix
.
data
(),
k
.
data
(),
prefix
.
size
());
if
(
c
==
0
)
{
keys
.
push_back
(
std
::
move
(
k
));
}
else
if
(
c
<
0
)
{
break
;
}
}
delete
it
;
return
keys
;
}
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
LevelDBDataStore
::
vlist_keyvals
(
const
ds_bulk_t
&
start_key
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
LevelDBDataStore
::
vlist_keyvals
(
const
ds_bulk_t
&
start
,
size_t
count
,
const
ds_bulk_t
&
prefix
)
{
std
::
vector
<
std
::
pair
<
ds_bulk_t
,
ds_bulk_t
>>
result
;
leveldb
::
Iterator
*
it
=
_dbm
->
NewIterator
(
leveldb
::
ReadOptions
());
leveldb
::
Slice
start_slice
(
start
.
data
(),
start
.
size
());
int
c
=
0
;
if
(
start
.
size
()
>
0
)
{
it
->
Seek
(
start_slice
);
/* we treat 'start' the way RADOS treats it: excluding it from returned
* keys. LevelDB treats start inclusively, so skip over it if we found
* an exact match */
if
(
start
.
size
()
==
it
->
key
().
size
()
&&
(
memcmp
(
it
->
key
().
data
(),
start
.
data
(),
start
.
size
())
==
0
))
it
->
Next
();
}
else
{
it
->
SeekToFirst
();
}
/* note: iterator initialized above, not in for loop */
for
(;
it
->
Valid
()
&&
result
.
size
()
<
count
;
it
->
Next
()
)
{
ds_bulk_t
k
(
it
->
key
().
size
());
ds_bulk_t
v
(
it
->
value
().
size
());
memcpy
(
k
.
data
(),
it
->
key
().
data
(),
it
->
key
().
size
());
memcpy
(
v
.
data
(),
it
->
value
().
data
(),
it
->
value
().
size
());
c
=
std
::
memcmp
(
prefix
.
data
(),
k
.
data
(),
prefix
.
size
());
if
(
c
==
0
)
{
result
.
push_back
(
std
::
make_pair
(
std
::
move
(
k
),
std
::
move
(
v
)));
}
else
if
(
c
<
0
)
{
break
;
}
}
delete
it
;
return
result
;
}
/*
{
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> keyvals;
...
...
@@ -172,3 +216,4 @@ std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(cons
delete it;
return keyvals;
}
*/
test/basic.sh
View file @
5f8918aa
...
...
@@ -28,7 +28,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/erase-test.sh
View file @
5f8918aa
...
...
@@ -27,7 +27,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/get-test.sh
View file @
5f8918aa
...
...
@@ -27,7 +27,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/length-test.sh
View file @
5f8918aa
...
...
@@ -27,7 +27,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/list-keys-prefix-test.sh
View file @
5f8918aa
...
...
@@ -27,7 +27,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/list-keys-test.sh
View file @
5f8918aa
...
...
@@ -27,7 +27,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/list-keyvals-test.sh
View file @
5f8918aa
...
...
@@ -16,7 +16,7 @@ sleep 1
#####################
run_to 20
test
/sdskv-list-keyvals-test
$svr_addr
1
$test_db_name
1
0
run_to 20
test
/sdskv-list-keyvals-test
$svr_addr
1
$test_db_name
3
0
if
[
$?
-ne
0
]
;
then
wait
exit
1
...
...
@@ -27,7 +27,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/open-test.sh
View file @
5f8918aa
...
...
@@ -28,7 +28,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/put-test.sh
View file @
5f8918aa
...
...
@@ -27,7 +27,4 @@ wait
echo
cleaning up
$TMPBASE
rm
-rf
$TMPBASE
echo
cleaning up
$test_db_name
rm_db
exit
0
test/sdskv-get-test.cc
View file @
5f8918aa
...
...
@@ -8,6 +8,7 @@
#include
<assert.h>
#include
<unistd.h>
#include
<margo.h>
#include
<iostream>
#include
<string>
#include
<vector>
#include
<algorithm>
...
...
@@ -102,12 +103,12 @@ int main(int argc, char *argv[])
/* **** put keys ***** */
std
::
vector
<
std
::
string
>
keys
;
std
::
map
<
std
::
string
,
std
::
string
>
reference
;
size_t
max_value_size
=
8000
;
size_t
max_value_size
=
24
;
for
(
unsigned
i
=
0
;
i
<
num_keys
;
i
++
)
{
auto
k
=
gen_random_string
(
16
);
// half of the entries will be put using bulk
auto
v
=
gen_random_string
(
i
*
max_value_size
/
num_keys
);
auto
v
=
gen_random_string
(
3
+
i
*
(
max_value_size
-
3
)
/
num_keys
);
ret
=
sdskv_put
(
kvph
,
db_id
,
(
const
void
*
)
k
.
data
(),
k
.
size
()
+
1
,
(
const
void
*
)
v
.
data
(),
v
.
size
()
+
1
);
...
...
@@ -120,6 +121,7 @@ int main(int argc, char *argv[])
margo_finalize
(
mid
);
return
-
1
;
}
std
::
cout
<<
"Inserted "
<<
k
<<
"
\t
===> "
<<
v
<<
std
::
endl
;
reference
[
k
]
=
v
;
keys
.
push_back
(
k
);
}
...
...
@@ -143,6 +145,7 @@ int main(int argc, char *argv[])
return
-
1
;
}
std
::
string
vstring
((
char
*
)(
v
.
data
()));
std
::
cout
<<
"Got "
<<
k
<<
" ===> "
<<
vstring
<<
std
::
endl
;
if
(
vstring
!=
reference
[
k
])
{
fprintf
(
stderr
,
"Error: sdskv_get() returned a value different from the reference
\n
"
);
sdskv_shutdown_service
(
kvcl
,
svr_addr
);
...
...
test/sdskv-list-kv-test.cc
View file @
5f8918aa
...
...
@@ -103,7 +103,7 @@ int main(int argc, char *argv[])
/* **** put keys ***** */
std
::
vector
<
std
::
string
>
keys
;
std
::
map
<
std
::
string
,
std
::
string
>
reference
;
size_t
max_value_size
=
16
;
size_t
max_value_size
=
8000
;
size_t
max_key_size
=
16
;
for
(
unsigned
i
=
0
;
i
<
num_keys
;
i
++
)
{
...
...
test/test-util.sh
View file @
5f8918aa
...
...
@@ -44,12 +44,7 @@ function test_start_server ()
function
find_db_name
()
{
test_db_name
=
${
SDSKV_TEST_DB_NAME
:-
"
/tmp/
sdskv-test-db"
}
test_db_name
=
$
TMPBASE
/
$
{
SDSKV_TEST_DB_NAME
:-
"sdskv-test-db"
}
test_db_type
=
${
SDSKV_TEST_DB_TYPE
:-
"map"
}
test_db_full
=
"
${
test_db_name
}
:
${
test_db_type
}
"
}
function
rm_db
()
{
rm
-rf
${
test_db_name
}
}
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment