Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
P
py-bake
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
sds
py-bake
Commits
8866767d
Commit
8866767d
authored
May 11, 2018
by
Matthieu Dorier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
added client-side functions
parent
cb20a717
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
451 additions
and
33 deletions
+451
-33
pybake/__init__.py
pybake/__init__.py
+0
-1
pybake/client.py
pybake/client.py
+180
-13
pybake/server.py
pybake/server.py
+2
-2
pybake/src/client.cpp
pybake/src/client.cpp
+104
-0
pybake/src/server.cpp
pybake/src/server.cpp
+5
-5
pybake/src/target.cpp
pybake/src/target.cpp
+61
-0
pybake/target.py
pybake/target.py
+32
-5
setup.py
setup.py
+11
-5
test/client.py
test/client.py
+54
-0
test/server.py
test/server.py
+2
-2
No files found.
pybake/__init__.py
View file @
8866767d
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
from
pybake.target
import
BakeStorageTarget
pybake/client.py
View file @
8866767d
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import
_pybakeclient
from
pybake.target
import
*
import
pymargo
class
BakeClient
():
"""
The BakeClient class wraps a bake_client_t structure at C level.
It registers the RPCs necessary to interact with a Bake provider.
It can be used to create provider handles pointing to Bake providers.
"""
def
__init__
(
self
,
mid
):
self
.
_client
=
_pybakeclient
.
client_init
(
mid
.
_mid
)
def
__init__
(
self
,
mid
):
"""
Constructor. Initializes a new BakeClient with a MargoInstance.
def
create_provider_handle
(
self
,
addr
,
mplex_id
):
ph
=
_pybakeclient
.
provider_handle_create
(
self
.
_client
,
addr
.
get_hg_addr
(),
mplex_id
)
return
BakeProviderHandle
(
ph
)
Args:
mid (MargoInstance): MargoInstance on which to register RPCs.
"""
self
.
_client
=
_pybakeclient
.
client_init
(
mid
.
_mid
)
def
shutdown_service
(
self
,
addr
):
_pybakeclient
.
shutdown_service
(
self
.
_client
,
addr
.
get_hg_addr
())
def
create_provider_handle
(
self
,
addr
,
provider_id
):
"""
Creates a BakeProviderHandle object pointing to the given
address and provider id.
Args:
addr (MargoAddress): Address of the Bake provider.
provider_id (int): ID of the provider.
"""
ph
=
_pybakeclient
.
provider_handle_create
(
self
.
_client
,
addr
.
get_hg_addr
(),
provider_id
)
return
BakeProviderHandle
(
ph
)
def
shutdown_service
(
self
,
addr
):
"""
Shut down a MargoInstance running at a particular address.
Args:
addr (MargoAddress): Address of the MargoInstance to shut down.
"""
_pybakeclient
.
shutdown_service
(
self
.
_client
,
addr
.
get_hg_addr
())
def
__del__
(
self
):
_pybakeclient
.
client_finalize
(
self
.
_client
)
def
finalize
(
self
):
"""
Finalizes the underlying bake_client_t structure.
"""
_pybakeclient
.
client_finalize
(
self
.
_client
)
class
BakeProviderHandle
():
"""
The BakeProviderHandle class represents a handle to a remote Bake provider.
Internally, this class wraps a bake_provider_handle_t C structure.
"""
def
__init__
(
self
,
ph
):
self
.
_ph
=
ph
def
__init__
(
self
,
ph
):
"""
Constructor. This is not supposed to be called by users.
Users should create a BakeProviderHandle from a BakeClient clt
by calling clt.create_provider_handle.
"""
self
.
_ph
=
ph
def
__del__
(
self
):
_pybakeclient
.
provider_handle_release
(
self
.
_ph
)
def
__del__
(
self
):
"""
Explicit destructor to call provider_handle_release on the underlying
bake_provider_handle_t C structure.
"""
_pybakeclient
.
provider_handle_release
(
self
.
_ph
)
def
get_eager_limit
(
self
):
"""
Get the threshold size bellow which this provider
handle will embed data within the RPC arguments.
"""
return
_pybakeclient
.
get_eager_limit
(
self
.
_ph
)
def
set_eager_limit
(
self
,
limit
):
"""
Set the threshold size bellow which this provider handle
will embed data within the RPC arguments.
"""
return
_pybakeclient
.
set_eager_limit
(
self
.
_ph
,
limit
)
def
probe
(
self
,
max_targets
=
0
):
"""
Get the list of BakeTargetIDs of targets located in
this provider. If max_targets is not specified, this
function will return all the target ids.
"""
if
(
max_targets
!=
0
):
tgts
=
_pybakeclient
.
probe
(
self
.
_ph
,
max_targets
)
else
:
num_targets
=
32
while
(
True
):
tgts
=
_pybakeclient
.
probe
(
self
.
_ph
,
num_targets
)
if
(
len
(
tgts
)
==
num_targets
):
num_targets
*=
2
else
:
break
result
=
[]
for
tgt
in
tgts
:
result
.
append
(
BakeTargetID
(
tgt
))
return
result
def
create
(
self
,
bti
,
region_size
):
"""
Creates a region in the specified target, with a given size.
Args:
bti (BakeTargetID): ID of the bake target in which to create the region.
region_size (int): size of the region to create.
Returns:
A BakeRegionID object representing the region.
"""
rid
=
_pybakeclient
.
create
(
self
.
_ph
,
bti
.
_tid
,
region_size
)
return
BakeRegionID
(
rid
)
def
write
(
self
,
rid
,
offset
,
data
):
"""
Writes data in a region, at a specified offset.
Args:
rid (BakeRegionID): region in which to write.
offset (int): offset at which to write.
data (str): data to write.
"""
return
_pybakeclient
.
write
(
self
.
_ph
,
rid
.
_rid
,
offset
,
data
)
def
persist
(
self
,
rid
):
"""
Make the changes to a given region persist.
Args:
rid (BakeRegionID): region to persist.
"""
return
_pybakeclient
.
persist
(
self
.
_ph
,
rid
.
_rid
)
def
create_write_persist
(
self
,
bti
,
data
):
"""
Creates a new region, write data to it at a given offset,
and persist the region.
Args:
bti (BakeTargetID): target id in which to create the region.
size (int): size of the region to create.
offset (int): offset at which to write data in the region.
data (str): data to write.
"""
rid
=
_pybakeclient
.
create_write_persist
(
self
.
_ph
,
bti
.
_tid
,
data
)
return
BakeRegionID
(
rid
)
def
get_size
(
self
,
rid
):
"""
Get the size of a given region.
Args:
rid (BakeRegionID): region id.
Returns:
The size (ind) of the provided region.
"""
return
_pybakeclient
.
get_size
(
self
.
_ph
,
rid
.
_rid
)
def
read
(
self
,
rid
,
offset
=
0
,
size
=-
1
):
"""
Reads the data contained in a given region, at a given offset
and with a given size. If the size is not provided,
this function will first send an RPC to get the current region size
and then read from the offset up to the end of the region. If the
offset is not provided, the entire region is read.
Args:
rid (BakeRegionID): region id.
offset (int): offset at which to read.
size (int): size to read.
Returns:
The data read, in the form of a string, or None if an
error occured.
"""
if
(
size
<
0
):
size
=
self
.
get_size
(
rid
)
-
offset
return
_pybakeclient
.
read
(
self
.
_ph
,
rid
.
_rid
,
offset
,
size
)
def
remove
(
self
,
rid
):
"""
Remove a region from its target.
Args:
rid (BakeRegionID): region to remove.
"""
_pybakeclient
.
remove
(
self
.
_ph
,
rid
.
_rid
)
pybake/server.py
View file @
8866767d
...
...
@@ -2,7 +2,7 @@
# See COPYRIGHT in top-level directory.
import
_pybakeserver
import
pymargo
from
pybake.target
import
Bake
StorageTarget
from
pybake.target
import
Bake
TargetID
def
make_pool
(
name
,
size
,
mode
):
_pybakeserver
.
make_pool
(
name
,
size
,
mode
)
...
...
@@ -15,7 +15,7 @@ class BakeProvider(pymargo.Provider):
def
add_storage_target
(
self
,
name
):
tid
=
_pybakeserver
.
add_storage_target
(
self
.
_provider
,
name
)
return
Bake
StorageTarget
(
tid
)
return
Bake
TargetID
(
tid
)
def
remove_storage_target
(
self
,
target
):
return
_pybakeserver
.
remove_storage_target
(
self
.
_provider
,
target
.
_tid
)
...
...
pybake/src/client.cpp
View file @
8866767d
...
...
@@ -42,17 +42,121 @@ static bake_provider_handle_t pybake_provider_handle_create(
return
providerHandle
;
}
static
uint64_t
pybake_get_eager_limit
(
bake_provider_handle_t
ph
)
{
uint64_t
limit
;
int
ret
=
bake_provider_handle_get_eager_limit
(
ph
,
&
limit
);
if
(
ret
!=
0
)
return
0
;
return
limit
;
}
static
bpl
::
object
pybake_probe
(
bake_provider_handle_t
ph
,
uint64_t
max_targets
)
{
bpl
::
list
result
;
std
::
vector
<
bake_target_id_t
>
targets
(
max_targets
);
uint64_t
num_targets
;
int
ret
=
bake_probe
(
ph
,
max_targets
,
targets
.
data
(),
&
num_targets
);
if
(
ret
!=
0
)
return
bpl
::
object
();
for
(
uint64_t
i
=
0
;
i
<
num_targets
;
i
++
)
{
result
.
append
(
bpl
::
object
(
targets
[
i
]));
}
return
result
;
}
static
bpl
::
object
pybake_create
(
bake_provider_handle_t
ph
,
bake_target_id_t
bti
,
size_t
region_size
)
{
bake_region_id_t
rid
;
std
::
memset
(
&
rid
,
0
,
sizeof
(
rid
));
int
ret
=
bake_create
(
ph
,
bti
,
region_size
,
&
rid
);
if
(
ret
!=
0
)
return
bpl
::
object
();
else
return
bpl
::
object
(
rid
);
}
static
bpl
::
object
pybake_write
(
bake_provider_handle_t
ph
,
const
bake_region_id_t
&
rid
,
uint64_t
offset
,
const
std
::
string
&
data
)
{
int
ret
=
bake_write
(
ph
,
rid
,
offset
,
(
const
void
*
)
data
.
data
(),
data
.
size
());
if
(
ret
==
0
)
return
bpl
::
object
(
true
);
else
return
bpl
::
object
(
false
);
}
static
bpl
::
object
pybake_persist
(
bake_provider_handle_t
ph
,
const
bake_region_id_t
&
rid
)
{
int
ret
=
bake_persist
(
ph
,
rid
);
if
(
ret
==
0
)
return
bpl
::
object
(
true
);
else
return
bpl
::
object
(
false
);
}
static
bpl
::
object
pybake_create_write_persist
(
bake_provider_handle_t
ph
,
bake_target_id_t
tid
,
const
std
::
string
&
data
)
{
bake_region_id_t
rid
;
int
ret
=
bake_create_write_persist
(
ph
,
tid
,
data
.
data
(),
data
.
size
(),
&
rid
);
if
(
ret
==
0
)
return
bpl
::
object
(
rid
);
else
return
bpl
::
object
();
}
static
bpl
::
object
pybake_get_size
(
bake_provider_handle_t
ph
,
const
bake_region_id_t
&
rid
)
{
uint64_t
size
;
int
ret
=
bake_get_size
(
ph
,
rid
,
&
size
);
if
(
ret
==
0
)
return
bpl
::
object
(
size
);
else
return
bpl
::
object
();
}
static
bpl
::
object
pybake_read
(
bake_provider_handle_t
ph
,
const
bake_region_id_t
&
rid
,
uint64_t
offset
,
size_t
size
)
{
std
::
string
result
(
size
,
'\0'
);
uint64_t
bytes_read
;
int
ret
=
bake_read
(
ph
,
rid
,
offset
,
(
void
*
)
result
.
data
(),
size
,
&
bytes_read
);
if
(
ret
!=
0
)
return
bpl
::
object
();
result
.
resize
(
bytes_read
);
return
bpl
::
object
(
result
);
}
BOOST_PYTHON_MODULE
(
_pybakeclient
)
{
#define ret_policy_opaque bpl::return_value_policy<bpl::return_opaque_pointer>()
bpl
::
import
(
"_pybaketarget"
);
bpl
::
opaque
<
bake_client
>
();
bpl
::
opaque
<
bake_provider_handle
>
();
// bpl::class_<bake_region_id_t>("bake_region_id", bpl::no_init);
bpl
::
def
(
"client_init"
,
&
pybake_client_init
,
ret_policy_opaque
);
bpl
::
def
(
"client_finalize"
,
&
bake_client_finalize
);
bpl
::
def
(
"provider_handle_create"
,
&
pybake_provider_handle_create
,
ret_policy_opaque
);
bpl
::
def
(
"provider_handle_ref_incr"
,
&
bake_provider_handle_ref_incr
);
bpl
::
def
(
"provider_handle_release"
,
&
bake_provider_handle_release
);
bpl
::
def
(
"get_eager_limit"
,
&
pybake_get_eager_limit
);
bpl
::
def
(
"set_eager_limit"
,
&
bake_provider_handle_set_eager_limit
);
bpl
::
def
(
"probe"
,
&
pybake_probe
);
bpl
::
def
(
"create"
,
&
pybake_create
);
bpl
::
def
(
"write"
,
&
pybake_write
);
bpl
::
def
(
"persist"
,
&
pybake_persist
);
bpl
::
def
(
"create_write_persist"
,
&
pybake_create_write_persist
);
bpl
::
def
(
"get_size"
,
&
pybake_get_size
);
bpl
::
def
(
"read"
,
&
pybake_read
);
bpl
::
def
(
"remove"
,
&
bake_remove
);
bpl
::
def
(
"shutdown_service"
,
&
bake_shutdown_service
);
#undef ret_policy_opaque
...
...
pybake/src/server.cpp
View file @
8866767d
...
...
@@ -40,13 +40,13 @@ static bake_target_id_t pybake_provider_add_storage_target(
provider
,
target_name
.
c_str
(),
&
target_id
);
return
target_id
;
}
#if 0
static std::string pybake_target_id_to_string(bake_target_id_t tid) {
char id[37];
uuid_unparse(tid.id, id);
return std::string(id);
}
#endif
static
bool
pybake_provider_remove_storage_target
(
bake_provider_t
provider
,
bake_target_id_t
target_id
)
...
...
@@ -90,10 +90,10 @@ BOOST_PYTHON_MODULE(_pybakeserver)
{
#define ret_policy_opaque bpl::return_value_policy<bpl::return_opaque_pointer>()
bpl
::
import
(
"_pybaketarget"
);
bpl
::
opaque
<
bake_server_context_t
>
();
bpl
::
class_
<
bake_target_id_t
>
(
"bake_target_id"
,
bpl
::
no_init
)
.
def
(
"__str__"
,
pybake_target_id_to_string
);
bpl
::
class_
<
bake_region_id_t
>
(
"bake_region_id"
,
bpl
::
no_init
);
// bpl::class_<bake_target_id_t>("bake_target_id", bpl::no_init)
// .def("__str__", pybake_target_id_to_string);
bpl
::
def
(
"register"
,
&
pybake_provider_register
,
ret_policy_opaque
);
bpl
::
def
(
"add_storage_target"
,
&
pybake_provider_add_storage_target
);
bpl
::
def
(
"remove_storage_target"
,
&
pybake_provider_remove_storage_target
);
...
...
pybake/src/target.cpp
0 → 100644
View file @
8866767d
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#define BOOST_NO_AUTO_PTR
#include <boost/python.hpp>
#include <boost/python/return_opaque_pointer.hpp>
#include <boost/python/handle.hpp>
#include <boost/python/enum.hpp>
#include <boost/python/def.hpp>
#include <boost/python/module.hpp>
#include <boost/python/return_value_policy.hpp>
#include <string>
#include <vector>
#include <cstring>
#include <iostream>
#include <margo.h>
#include <bake.h>
#include <bake-server.h>
namespace
bpl
=
boost
::
python
;
static
std
::
string
pybake_target_id_to_string
(
bake_target_id_t
tid
)
{
char
id
[
37
];
uuid_unparse
(
tid
.
id
,
id
);
return
std
::
string
(
id
);
}
static
bpl
::
object
pybake_target_id_from_string
(
const
std
::
string
&
tidstr
)
{
bake_target_id_t
tid
;
memset
(
tid
.
id
,
0
,
sizeof
(
uuid_t
));
if
(
tidstr
.
size
()
!=
36
)
return
bpl
::
object
();
int
ret
=
uuid_parse
((
char
*
)
tidstr
.
c_str
(),
tid
.
id
);
if
(
ret
==
0
)
return
bpl
::
object
(
tid
);
else
return
bpl
::
object
();
}
static
std
::
string
pybake_region_id_to_string
(
const
bake_region_id_t
&
region_id
)
{
std
::
string
result
((
const
char
*
)(
&
region_id
),
sizeof
(
region_id
));
return
result
;
}
static
bpl
::
object
pybake_region_id_from_string
(
const
std
::
string
&
region_str
)
{
bake_region_id_t
result
;
memset
(
&
result
,
0
,
sizeof
(
result
));
if
(
region_str
.
size
()
!=
sizeof
(
bake_region_id_t
))
return
bpl
::
object
();
memcpy
(
&
result
,
region_str
.
data
(),
sizeof
(
bake_region_id_t
));
return
bpl
::
object
(
result
);
}
BOOST_PYTHON_MODULE
(
_pybaketarget
)
{
bpl
::
class_
<
bake_target_id_t
>
(
"bake_target_id"
,
bpl
::
no_init
)
.
def
(
"__str__"
,
pybake_target_id_to_string
);
bpl
::
def
(
"target_id_from_string"
,
pybake_target_id_from_string
);
bpl
::
class_
<
bake_region_id_t
>
(
"bake_region_id"
,
bpl
::
no_init
)
.
def
(
"__str__"
,
pybake_region_id_to_string
);
bpl
::
def
(
"region_id_from_string"
,
pybake_region_id_from_string
);
}
pybake/target.py
View file @
8866767d
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
class
BakeStorageTarget
():
import
_pybaketarget
import
base64
def
__init__
(
self
,
tid
):
self
.
__tid
=
tid
class
BakeTargetID
():
def
__str__
(
self
):
return
str
(
self
.
__tid
)
def
__init__
(
self
,
tid
):
self
.
_tid
=
tid
def
__str__
(
self
):
return
str
(
self
.
_tid
)
@
staticmethod
def
from_str
(
string
):
tid
=
_pybaketarget
.
target_id_from_string
(
string
)
if
(
tid
is
None
):
return
None
else
:
return
BakeTargetID
(
tid
)
class
BakeRegionID
():
def
__init__
(
self
,
rid
):
self
.
_rid
=
rid
def
__str__
(
self
):
return
base64
.
b64encode
(
str
(
self
.
_rid
))
@
staticmethod
def
from_str
(
string
):
rid
=
_pybaketarget
.
region_id_from_string
(
base64
.
b64decode
(
string
))
if
(
rid
is
None
):
return
None
else
:
return
BakeRegionID
(
rid
)
setup.py
View file @
8866767d
...
...
@@ -12,17 +12,23 @@ os.environ['OPT'] = " ".join(
pybake_server_module
=
Extension
(
'_pybakeserver'
,
[
"pybake/src/server.cpp"
],
libraries
=
[
'boost_python'
,
'margo'
,
'bake-server'
],
include_dirs
=
[
'.'
],
depends
=
[])
include_dirs
=
[
'.'
],
depends
=
[])
pybake_client_module
=
Extension
(
'_pybakeclient'
,
[
"pybake/src/client.cpp"
],
libraries
=
[
'boost_python'
,
'margo'
,
'bake-client'
],
include_dirs
=
[
'.'
],
depends
=
[])
include_dirs
=
[
'.'
],
depends
=
[])
pybake_target_module
=
Extension
(
'_pybaketarget'
,
[
"pybake/src/target.cpp"
],
libraries
=
[
'boost_python'
,
'uuid'
],
include_dirs
=
[
'.'
],
depends
=
[])
setup
(
name
=
'pybake'
,
version
=
'0.1'
,
author
=
'Matthieu Dorier'
,
description
=
"""Python binding for BAKE"""
,
ext_modules
=
[
pybake_server_module
,
pybake_client_module
],
ext_modules
=
[
pybake_server_module
,
pybake_client_module
,
pybake_target_module
],
packages
=
[
'pybake'
]
)
test/client.py
0 → 100644
View file @
8866767d
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import
sys
from
pymargo
import
MargoInstance
from
pybake.target
import
BakeRegionID
from
pybake.client
import
*
mid
=
MargoInstance
(
'tcp'
)
server_addr
=
sys
.
argv
[
1
]
mplex_id
=
int
(
sys
.
argv
[
2
])
client
=
BakeClient
(
mid
)
addr
=
mid
.
lookup
(
server_addr
)
ph
=
client
.
create_provider_handle
(
addr
,
mplex_id
)
# Testing get_eager_limit
lim
=
ph
.
get_eager_limit
()
print
"Eager limit is: "
+
str
(
lim
)
# probe the provider handle (for all targets)
targets
=
ph
.
probe
()
print
"Probe found the following targets:"
for
t
in
targets
:
print
"===== "
+
str
(
t
)
target
=
targets
[
0
]
# create a 32-bytes region in the first target
region
=
ph
.
create
(
target
,
32
)
print
"Created region "
+
str
(
region
)
regionstr
=
str
(
region
)
region
=
BakeRegionID
.
from_str
(
regionstr
)
# write into the region
ph
.
write
(
region
,
0
,
'A'
*
16
)
ph
.
write
(
region
,
16
,
'B'
*
16
)
# get size of region
s
=
ph
.
get_size
(
region
)
print
"Region size is "
+
str
(
s
)
# persist region
ph
.
persist
(
region
)
# read region
result
=
ph
.
read
(
region
,
8
,
16
)
print
"Reading region at offset 8, size 16 gives: "
+
str
(
result
)
del
ph
client
.
shutdown_service
(
addr
)
del
addr
client
.
finalize
()
mid
.
finalize
()
test/server.py
View file @
8866767d
...
...
@@ -25,7 +25,7 @@ targets = provider.list_storage_targets()
for
t
in
targets
:
print
str
(
t
)
provider
.
remove_all_storage_targets
()
print
"number of targets: "
+
str
(
provider
.
count_storage_targets
())
#
provider.remove_all_storage_targets()
#
print "number of targets: "+str(provider.count_storage_targets())
mid
.
wait_for_finalize
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment