Commit 8866767d authored by Matthieu Dorier's avatar Matthieu Dorier

added client-side functions

parent cb20a717
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
from pybake.target import BakeStorageTarget
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import _pybakeclient
from pybake.target import *
import pymargo
class BakeClient():
"""
The BakeClient class wraps a bake_client_t structure at C level.
It registers the RPCs necessary to interact with a Bake provider.
It can be used to create provider handles pointing to Bake providers.
"""
def __init__(self, mid):
self._client = _pybakeclient.client_init(mid._mid)
def __init__(self, mid):
"""
Constructor. Initializes a new BakeClient with a MargoInstance.
def create_provider_handle(self, addr, mplex_id):
ph = _pybakeclient.provider_handle_create(self._client, addr.get_hg_addr(), mplex_id)
return BakeProviderHandle(ph)
Args:
mid (MargoInstance): MargoInstance on which to register RPCs.
"""
self._client = _pybakeclient.client_init(mid._mid)
def shutdown_service(self, addr):
_pybakeclient.shutdown_service(self._client, addr.get_hg_addr())
def create_provider_handle(self, addr, provider_id):
"""
Creates a BakeProviderHandle object pointing to the given
address and provider id.
Args:
addr (MargoAddress): Address of the Bake provider.
provider_id (int): ID of the provider.
"""
ph = _pybakeclient.provider_handle_create(self._client, addr.get_hg_addr(), provider_id)
return BakeProviderHandle(ph)
def shutdown_service(self, addr):
"""
Shut down a MargoInstance running at a particular address.
Args:
addr (MargoAddress): Address of the MargoInstance to shut down.
"""
_pybakeclient.shutdown_service(self._client, addr.get_hg_addr())
def __del__(self):
_pybakeclient.client_finalize(self._client)
def finalize(self):
"""
Finalizes the underlying bake_client_t structure.
"""
_pybakeclient.client_finalize(self._client)
class BakeProviderHandle():
"""
The BakeProviderHandle class represents a handle to a remote Bake provider.
Internally, this class wraps a bake_provider_handle_t C structure.
"""
def __init__(self, ph):
self._ph = ph
def __init__(self, ph):
"""
Constructor. This is not supposed to be called by users.
Users should create a BakeProviderHandle from a BakeClient clt
by calling clt.create_provider_handle.
"""
self._ph = ph
def __del__(self):
_pybakeclient.provider_handle_release(self._ph)
def __del__(self):
"""
Explicit destructor to call provider_handle_release on the underlying
bake_provider_handle_t C structure.
"""
_pybakeclient.provider_handle_release(self._ph)
def get_eager_limit(self):
"""
Get the threshold size bellow which this provider
handle will embed data within the RPC arguments.
"""
return _pybakeclient.get_eager_limit(self._ph)
def set_eager_limit(self, limit):
"""
Set the threshold size bellow which this provider handle
will embed data within the RPC arguments.
"""
return _pybakeclient.set_eager_limit(self._ph, limit)
def probe(self, max_targets=0):
"""
Get the list of BakeTargetIDs of targets located in
this provider. If max_targets is not specified, this
function will return all the target ids.
"""
if(max_targets != 0):
tgts = _pybakeclient.probe(self._ph, max_targets)
else:
num_targets = 32
while(True):
tgts = _pybakeclient.probe(self._ph, num_targets)
if(len(tgts) == num_targets):
num_targets *= 2
else:
break
result = []
for tgt in tgts:
result.append(BakeTargetID(tgt))
return result
def create(self, bti, region_size):
"""
Creates a region in the specified target, with a given size.
Args:
bti (BakeTargetID): ID of the bake target in which to create the region.
region_size (int): size of the region to create.
Returns:
A BakeRegionID object representing the region.
"""
rid = _pybakeclient.create(self._ph, bti._tid, region_size)
return BakeRegionID(rid)
def write(self, rid, offset, data):
"""
Writes data in a region, at a specified offset.
Args:
rid (BakeRegionID): region in which to write.
offset (int): offset at which to write.
data (str): data to write.
"""
return _pybakeclient.write(self._ph, rid._rid, offset, data)
def persist(self, rid):
"""
Make the changes to a given region persist.
Args:
rid (BakeRegionID): region to persist.
"""
return _pybakeclient.persist(self._ph, rid._rid)
def create_write_persist(self, bti, data):
"""
Creates a new region, write data to it at a given offset,
and persist the region.
Args:
bti (BakeTargetID): target id in which to create the region.
size (int): size of the region to create.
offset (int): offset at which to write data in the region.
data (str): data to write.
"""
rid = _pybakeclient.create_write_persist(self._ph, bti._tid, data)
return BakeRegionID(rid)
def get_size(self, rid):
"""
Get the size of a given region.
Args:
rid (BakeRegionID): region id.
Returns:
The size (ind) of the provided region.
"""
return _pybakeclient.get_size(self._ph, rid._rid)
def read(self, rid, offset=0, size=-1):
"""
Reads the data contained in a given region, at a given offset
and with a given size. If the size is not provided,
this function will first send an RPC to get the current region size
and then read from the offset up to the end of the region. If the
offset is not provided, the entire region is read.
Args:
rid (BakeRegionID): region id.
offset (int): offset at which to read.
size (int): size to read.
Returns:
The data read, in the form of a string, or None if an
error occured.
"""
if(size < 0):
size = self.get_size(rid) - offset
return _pybakeclient.read(self._ph, rid._rid, offset, size)
def remove(self, rid):
"""
Remove a region from its target.
Args:
rid (BakeRegionID): region to remove.
"""
_pybakeclient.remove(self._ph, rid._rid)
......@@ -2,7 +2,7 @@
# See COPYRIGHT in top-level directory.
import _pybakeserver
import pymargo
from pybake.target import BakeStorageTarget
from pybake.target import BakeTargetID
def make_pool(name, size, mode):
_pybakeserver.make_pool(name, size, mode)
......@@ -15,7 +15,7 @@ class BakeProvider(pymargo.Provider):
def add_storage_target(self, name):
tid = _pybakeserver.add_storage_target(self._provider, name)
return BakeStorageTarget(tid)
return BakeTargetID(tid)
def remove_storage_target(self, target):
return _pybakeserver.remove_storage_target(self._provider, target._tid)
......
......@@ -42,17 +42,121 @@ static bake_provider_handle_t pybake_provider_handle_create(
return providerHandle;
}
static uint64_t pybake_get_eager_limit(
bake_provider_handle_t ph)
{
uint64_t limit;
int ret = bake_provider_handle_get_eager_limit(ph, &limit);
if(ret != 0) return 0;
return limit;
}
static bpl::object pybake_probe(
bake_provider_handle_t ph,
uint64_t max_targets)
{
bpl::list result;
std::vector<bake_target_id_t> targets(max_targets);
uint64_t num_targets;
int ret = bake_probe(ph, max_targets, targets.data(), &num_targets);
if(ret != 0) return bpl::object();
for(uint64_t i=0; i < num_targets; i++) {
result.append(bpl::object(targets[i]));
}
return result;
}
static bpl::object pybake_create(
bake_provider_handle_t ph,
bake_target_id_t bti,
size_t region_size)
{
bake_region_id_t rid;
std::memset(&rid, 0, sizeof(rid));
int ret = bake_create(ph, bti, region_size, &rid);
if(ret != 0) return bpl::object();
else return bpl::object(rid);
}
static bpl::object pybake_write(
bake_provider_handle_t ph,
const bake_region_id_t& rid,
uint64_t offset,
const std::string& data)
{
int ret = bake_write(ph, rid, offset, (const void*)data.data(), data.size());
if(ret == 0) return bpl::object(true);
else return bpl::object(false);
}
static bpl::object pybake_persist(
bake_provider_handle_t ph,
const bake_region_id_t& rid)
{
int ret = bake_persist(ph, rid);
if(ret == 0) return bpl::object(true);
else return bpl::object(false);
}
static bpl::object pybake_create_write_persist(
bake_provider_handle_t ph,
bake_target_id_t tid,
const std::string& data)
{
bake_region_id_t rid;
int ret = bake_create_write_persist(ph, tid,
data.data(), data.size(), &rid);
if(ret == 0) return bpl::object(rid);
else return bpl::object();
}
static bpl::object pybake_get_size(
bake_provider_handle_t ph,
const bake_region_id_t& rid)
{
uint64_t size;
int ret = bake_get_size(ph, rid, &size);
if(ret == 0) return bpl::object(size);
else return bpl::object();
}
static bpl::object pybake_read(
bake_provider_handle_t ph,
const bake_region_id_t& rid,
uint64_t offset,
size_t size)
{
std::string result(size, '\0');
uint64_t bytes_read;
int ret = bake_read(ph, rid, offset, (void*)result.data(), size, &bytes_read);
if(ret != 0) return bpl::object();
result.resize(bytes_read);
return bpl::object(result);
}
BOOST_PYTHON_MODULE(_pybakeclient)
{
#define ret_policy_opaque bpl::return_value_policy<bpl::return_opaque_pointer>()
bpl::import("_pybaketarget");
bpl::opaque<bake_client>();
bpl::opaque<bake_provider_handle>();
// bpl::class_<bake_region_id_t>("bake_region_id", bpl::no_init);
bpl::def("client_init", &pybake_client_init, ret_policy_opaque);
bpl::def("client_finalize", &bake_client_finalize);
bpl::def("provider_handle_create", &pybake_provider_handle_create, ret_policy_opaque);
bpl::def("provider_handle_ref_incr", &bake_provider_handle_ref_incr);
bpl::def("provider_handle_release", &bake_provider_handle_release);
bpl::def("get_eager_limit", &pybake_get_eager_limit);
bpl::def("set_eager_limit", &bake_provider_handle_set_eager_limit);
bpl::def("probe", &pybake_probe);
bpl::def("create", &pybake_create);
bpl::def("write", &pybake_write);
bpl::def("persist", &pybake_persist);
bpl::def("create_write_persist", &pybake_create_write_persist);
bpl::def("get_size", &pybake_get_size);
bpl::def("read", &pybake_read);
bpl::def("remove", &bake_remove);
bpl::def("shutdown_service", &bake_shutdown_service);
#undef ret_policy_opaque
......
......@@ -40,13 +40,13 @@ static bake_target_id_t pybake_provider_add_storage_target(
provider, target_name.c_str(), &target_id);
return target_id;
}
#if 0
static std::string pybake_target_id_to_string(bake_target_id_t tid) {
char id[37];
uuid_unparse(tid.id, id);
return std::string(id);
}
#endif
static bool pybake_provider_remove_storage_target(
bake_provider_t provider,
bake_target_id_t target_id)
......@@ -90,10 +90,10 @@ BOOST_PYTHON_MODULE(_pybakeserver)
{
#define ret_policy_opaque bpl::return_value_policy<bpl::return_opaque_pointer>()
bpl::import("_pybaketarget");
bpl::opaque<bake_server_context_t>();
bpl::class_<bake_target_id_t>("bake_target_id", bpl::no_init)
.def("__str__", pybake_target_id_to_string);
bpl::class_<bake_region_id_t>("bake_region_id", bpl::no_init);
// bpl::class_<bake_target_id_t>("bake_target_id", bpl::no_init)
// .def("__str__", pybake_target_id_to_string);
bpl::def("register", &pybake_provider_register, ret_policy_opaque);
bpl::def("add_storage_target", &pybake_provider_add_storage_target);
bpl::def("remove_storage_target", &pybake_provider_remove_storage_target);
......
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#define BOOST_NO_AUTO_PTR
#include <boost/python.hpp>
#include <boost/python/return_opaque_pointer.hpp>
#include <boost/python/handle.hpp>
#include <boost/python/enum.hpp>
#include <boost/python/def.hpp>
#include <boost/python/module.hpp>
#include <boost/python/return_value_policy.hpp>
#include <string>
#include <vector>
#include <cstring>
#include <iostream>
#include <margo.h>
#include <bake.h>
#include <bake-server.h>
namespace bpl = boost::python;
static std::string pybake_target_id_to_string(bake_target_id_t tid) {
char id[37];
uuid_unparse(tid.id, id);
return std::string(id);
}
static bpl::object pybake_target_id_from_string(const std::string& tidstr) {
bake_target_id_t tid;
memset(tid.id, 0, sizeof(uuid_t));
if(tidstr.size() != 36) return bpl::object();
int ret = uuid_parse((char*)tidstr.c_str(), tid.id);
if(ret == 0) return bpl::object(tid);
else return bpl::object();
}
static std::string pybake_region_id_to_string(const bake_region_id_t& region_id) {
std::string result((const char*)(&region_id), sizeof(region_id));
return result;
}
static bpl::object pybake_region_id_from_string(const std::string& region_str) {
bake_region_id_t result;
memset(&result, 0, sizeof(result));
if(region_str.size() != sizeof(bake_region_id_t))
return bpl::object();
memcpy(&result, region_str.data(), sizeof(bake_region_id_t));
return bpl::object(result);
}
BOOST_PYTHON_MODULE(_pybaketarget)
{
bpl::class_<bake_target_id_t>("bake_target_id", bpl::no_init)
.def("__str__", pybake_target_id_to_string);
bpl::def("target_id_from_string", pybake_target_id_from_string);
bpl::class_<bake_region_id_t>("bake_region_id", bpl::no_init)
.def("__str__", pybake_region_id_to_string);
bpl::def("region_id_from_string", pybake_region_id_from_string);
}
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
class BakeStorageTarget():
import _pybaketarget
import base64
def __init__(self, tid):
self.__tid = tid
class BakeTargetID():
def __str__(self):
return str(self.__tid)
def __init__(self, tid):
self._tid = tid
def __str__(self):
return str(self._tid)
@staticmethod
def from_str(string):
tid = _pybaketarget.target_id_from_string(string)
if(tid is None):
return None
else:
return BakeTargetID(tid)
class BakeRegionID():
def __init__(self, rid):
self._rid = rid
def __str__(self):
return base64.b64encode(str(self._rid))
@staticmethod
def from_str(string):
rid = _pybaketarget.region_id_from_string(base64.b64decode(string))
if(rid is None):
return None
else:
return BakeRegionID(rid)
......@@ -12,17 +12,23 @@ os.environ['OPT'] = " ".join(
pybake_server_module = Extension('_pybakeserver', ["pybake/src/server.cpp"],
libraries=['boost_python','margo','bake-server'],
include_dirs=['.'],
depends=[])
include_dirs=['.'],
depends=[])
pybake_client_module = Extension('_pybakeclient', ["pybake/src/client.cpp"],
libraries=['boost_python','margo','bake-client'],
include_dirs=['.'],
depends=[])
include_dirs=['.'],
depends=[])
pybake_target_module = Extension('_pybaketarget', ["pybake/src/target.cpp"],
libraries=['boost_python', 'uuid' ],
include_dirs=['.'],
depends=[])
setup(name='pybake',
version='0.1',
author='Matthieu Dorier',
description="""Python binding for BAKE""",
ext_modules=[ pybake_server_module, pybake_client_module ],
ext_modules=[ pybake_server_module, pybake_client_module, pybake_target_module ],
packages=['pybake']
)
# (C) 2018 The University of Chicago
# See COPYRIGHT in top-level directory.
import sys
from pymargo import MargoInstance
from pybake.target import BakeRegionID
from pybake.client import *
mid = MargoInstance('tcp')
server_addr = sys.argv[1]
mplex_id = int(sys.argv[2])
client = BakeClient(mid)
addr = mid.lookup(server_addr)
ph = client.create_provider_handle(addr, mplex_id)
# Testing get_eager_limit
lim = ph.get_eager_limit()
print "Eager limit is: "+str(lim)
# probe the provider handle (for all targets)
targets = ph.probe()
print "Probe found the following targets:"
for t in targets:
print "===== "+str(t)
target = targets[0]
# create a 32-bytes region in the first target
region = ph.create(target, 32)
print "Created region "+str(region)
regionstr = str(region)
region = BakeRegionID.from_str(regionstr)
# write into the region
ph.write(region, 0, 'A'*16)
ph.write(region, 16, 'B'*16)
# get size of region
s = ph.get_size(region)
print "Region size is "+str(s)
# persist region
ph.persist(region)
# read region
result = ph.read(region, 8, 16)
print "Reading region at offset 8, size 16 gives: "+str(result)
del ph
client.shutdown_service(addr)
del addr
client.finalize()
mid.finalize()
......@@ -25,7 +25,7 @@ targets = provider.list_storage_targets()
for t in targets:
print str(t)
provider.remove_all_storage_targets()
print "number of targets: "+str(provider.count_storage_targets())
#provider.remove_all_storage_targets()
#print "number of targets: "+str(provider.count_storage_targets())
mid.wait_for_finalize()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment