GitLab maintenance scheduled for Tomorrow, 2019-04-24, from 12:00 to 13:00 CDT - Services will be unavailable during this time.

Commit 0e6bae54 authored by Matthieu Dorier's avatar Matthieu Dorier

added proxy functions and more comments

parent c2a9e75b
......@@ -80,7 +80,7 @@ class BakeProviderHandle():
Set the threshold size bellow which this provider handle
will embed data within the RPC arguments.
"""
return _pybakeclient.set_eager_limit(self._ph, limit)
_pybakeclient.set_eager_limit(self._ph, limit)
def probe(self, max_targets=0):
"""
......@@ -115,8 +115,6 @@ class BakeProviderHandle():
A BakeRegionID object representing the region. None if an error occured.
"""
rid = _pybakeclient.create(self._ph, bti._tid, region_size)
if(rid == None):
return None
return BakeRegionID(rid)
def write(self, rid, offset, data):
......@@ -127,10 +125,8 @@ class BakeProviderHandle():
rid (BakeRegionID): region in which to write.
offset (int): offset at which to write.
data (str): data to write.
Returns:
True if the data was correctly written, False otherwise.
"""
return _pybakeclient.write(self._ph, rid._rid, offset, data)
_pybakeclient.write(self._ph, rid._rid, offset, data)
def write_numpy(self, rid, offset, array):
"""
......@@ -140,21 +136,39 @@ class BakeProviderHandle():
rid (BakeRegionID): region in which to write.
offset (int): offset at which to write.
data (numpy.ndarray): numpy array to write.
Returns:
True if the data was correctly written, False otherwise.
"""
return _pybakeclient.write_numpy(self._ph, rid._rid, offset, array)
_pybakeclient.write_numpy(self._ph, rid._rid, offset, array)
def proxy_write(self, rid, bulk, size, offset_in_region=0, offset_in_bulk=0, remote_addr=''):
"""
Writes data that is already exposed in a pymargo.bulk.Bulk object.
Args:
rid (BakeRegionID): region in which to write.
bulk (Bulk): bulk handle from which to get the data.
size (int): size to write.
offset_in_region (int): offset at which to write in the region.
offset_in_bulk (int): offset from which to read in the bulk object.
remote_addr (str): address of the process that created the bulk object.
"""
_pybakeclient.proxy_write(self._ph, rid._rid, offset_in_region,
bulk._hg_bulk, offset_in_bulk, remote_addr, size)
def persist(self, rid):
def persist(self, rid, offset=0, size=-1):
"""
Make the changes to a given region persist.
Offset is 0 by default. If size is not provided, PyBake will
call get_size to get it, which will throw an exception if Bake
has not been compiled with --enable-sizecheck.
Args:
rid (BakeRegionID): region to persist.
Returns:
True if the region was correctly persisted, False otherwise.
offset (int): offset in the region.
size (int): number of bytes to persist.
"""
return _pybakeclient.persist(self._ph, rid._rid)
if(size < 0):
size = self.get_size(rid) - offset
_pybakeclient.persist(self._ph, rid._rid, offset, size)
def create_write_persist(self, bti, data):
"""
......@@ -167,11 +181,27 @@ class BakeProviderHandle():
offset (int): offset at which to write data in the region.
data (str): data to write.
Returns:
True if the region was correctly written and persisted, False otherwise.
The created BakeRegionID.
"""
rid = _pybakeclient.create_write_persist(self._ph, bti._tid, data)
if(rid is None):
return None
return BakeRegionID(rid)
def proxy_create_write_persist(self, bti, bulk, size, offset_in_bulk=0, remote_addr=''):
"""
Version of create_write_persist that takes an already created bulk handle
as argument as well as the address of the process that created it.
Args:
bti (BakeTargetID): target id in which to create the region.
bulk (Bulk): bulk handle.
size (int): size of the region to create.
offset_in_bulk (int): offset in the bulk data.
remote_addr (str): Address of the process that created the bulk.
Returns:
The created BakeRegionID.
"""
rid = _pybakeclient.create_write_persist_proxy(self._ph,
bti._tid, bulk._hg_bulk, offset_in_bulk, remote_addr, size)
return BakeRegionID(rid)
def create_write_persist_numpy(self, bti, array):
......@@ -185,22 +215,21 @@ class BakeProviderHandle():
offset (int): offset at which to write data in the region.
array (numpy.ndarray): numpy array to write.
Returns:
True if the region was correctly written and persisted, False otherwise.
The created BakeRegionID.
"""
rid = _pybakeclient.create_write_persist_numpy(self._ph, bti._tid, array)
if(rid is None):
return None
return BakeRegionID(rid)
def get_size(self, rid):
"""
Get the size of a given region.
Get the size of a given region. Note thate bake should have been compiler
with --enable-sizecheck, otherwise this function will throw an exception.
Args:
rid (BakeRegionID): region id.
Returns:
The size (int) of the provided region. None in case of error.
The size (int) of the provided region.
"""
return _pybakeclient.get_size(self._ph, rid._rid)
......@@ -212,6 +241,10 @@ class BakeProviderHandle():
and then read from the offset up to the end of the region. If the
offset is not provided, the entire region is read.
Note that if bake has not been compiled with --enable-sizecheck,
then not providing the size argument will make this function throw
an exception.
Args:
rid (BakeRegionID): region id.
offset (int): offset at which to read.
......@@ -224,7 +257,24 @@ class BakeProviderHandle():
if(size < 0):
size = self.get_size(rid) - offset
return _pybakeclient.read(self._ph, rid._rid, offset, size)
def proxy_read(self, rid, bulk, size, offset_in_region=0, offset_in_bulk=0, remote_addr=''):
"""
Reads the data contained in a given region and pushes it to a provided bulk handle.
Args:
rid (BakeRegionID): region id.
bulk (Bulk): bulk handle where to push the data.
size (int): size to read.
offset_in_region (int): offset in the region where to start reading.
offset_in_bulk (int): offset in the bulk where to start placing data.
remote_addr (str): address of the process that created the Bulk object.
Returns:
the effective number of bytes read.
"""
return _pybakeclient.proxy_read(self._ph, rid._rid, offset_in_region, bulk._hg_bulk,
offset_in_bulk, remote_addr, size)
def read_numpy(self, rid, offset, shape, dtype):
"""
Reads the data contained in a given region, at a given offset,
......@@ -251,8 +301,7 @@ class BakeProviderHandle():
Args:
rid (BakeRegionID): region to remove.
"""
ret = _pybakeclient.remove(self._ph, rid._rid)
return (ret == 0)
_pybakeclient.remove(self._ph, rid._rid)
def migrate_region(self, source_rid, dest_addr, dest_provider_id, dest_target, remove_source=True):
"""
......@@ -271,8 +320,6 @@ class BakeProviderHandle():
"""
ret = _pybakeclient.migrate_region(self._ph, source_rid._rid, remove_source,
str(dest_addr), int(dest_provider_id), dest_target._tid)
if(ret == None):
return ret
return BakeRegionID(ret)
def migrate_target(self, source_tid, dest_addr, dest_provider_id, dest_root, remove_source=True):
......@@ -290,5 +337,5 @@ class BakeProviderHandle():
Returns:
True if the target was correctly migrated.
"""
return _pybakeclient.migrate_target(self._ph, source_tid._tid, remove_source,
_pybakeclient.migrate_target(self._ph, source_tid._tid, remove_source,
str(dest_addr), int(dest_provider_id), str(dest_root))
......@@ -9,27 +9,63 @@ def make_pool(name, size, mode):
_pybakeserver.make_pool(name, size, mode)
class BakeProvider(pymargo.Provider):
"""
The BakeProvide class wraps a C-level bake_provider_t object.
"""
def __init__(self, mid, provider_id):
super(BakeProvider, self).__init__(mid, provider_id)
def __init__(self, engine, provider_id):
"""
Constructor. Initializes a provider with an Engine and provider_id.
"""
super(BakeProvider, self).__init__(engine, provider_id)
self._provider = _pybakeserver.register(mid._mid, provider_id)
def add_storage_target(self, name):
tid = _pybakeserver.add_storage_target(self._provider, name)
def add_storage_target(self, path):
"""
Adds a storage target to the provider.
Returns a BakeTargetID instance that can be used to access the storage target.
"""
tid = _pybakeserver.add_storage_target(self._provider, path)
return BakeTargetID(tid)
def remove_storage_target(self, target):
return _pybakeserver.remove_storage_target(self._provider, target._tid)
"""
Removes a storage target from the provider. This does not delete the underlying file.
The target argument must be a BakeTargetID object.
"""
_pybakeserver.remove_storage_target(self._provider, target._tid)
def remove_all_storage_targets(self):
return _pybakeserver.remove_all_storage_targets(self._provider)
"""
Removes all the storage targets managed by this provider.
"""
_pybakeserver.remove_all_storage_targets(self._provider)
def count_storage_targets(self):
"""
Returns the number of storage targets that this provider manages.
"""
return _pybakeserver.count_storage_targets(self._provider)
def list_storage_targets(self):
"""
Returns the list of storage targets (BakeTargetIDs) that this provider manages.
"""
l = _pybakeserver.list_storage_targets(self._provider)
if(l is None):
return []
else:
return [ BakeTargetID(tid) for tid in l ]
def set_target_xfer_buffer(self, target, count, size):
"""
Sets the number and size of intermediate buffers that can be used to
execute transfers to a specific target.
"""
_pybakeserver.set_target_xfer_buffer(self._provider, target._tid, count, size)
def set_target_xfer_concurrency(self, target, num_threads):
"""
Sets the number of ULTs that can be used to execute transfers concurrently.
"""
_pybakeserver.set_target_xfer_concurrency(self._provider, target._tid, num_threads)
This diff is collapsed.
......@@ -4,6 +4,8 @@
* See COPYRIGHT in top-level directory.
*/
#include <pybind11/pybind11.h>
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
#include <cstring>
......@@ -14,6 +16,14 @@
namespace py11 = pybind11;
#define HANDLE_ERROR(__func, __ret) do {\
if(__ret != BAKE_SUCCESS) {\
std::stringstream ss;\
ss << #__func << "() failed (ret = " << __ret << ")";\
throw std::runtime_error(ss.str());\
}\
} while(0)
typedef py11::capsule pymargo_instance_id;
typedef py11::capsule pymargo_addr;
typedef py11::capsule pybake_provider_t;
......@@ -25,8 +35,8 @@ typedef py11::capsule pybake_provider_t;
static pybake_provider_t pybake_provider_register(pymargo_instance_id mid, uint8_t provider_id) {
bake_provider_t provider;
int ret = bake_provider_register(mid, provider_id, BAKE_ABT_POOL_DEFAULT, &provider);
if(ret != 0) return py11::none();
else return BAKEPR2CAPSULE(provider);
HANDLE_ERROR(bake_provider_register, ret);
return BAKEPR2CAPSULE(provider);
}
static py11::object pybake_provider_add_storage_target(
......@@ -36,30 +46,31 @@ static py11::object pybake_provider_add_storage_target(
std::memset(&target_id, 0, sizeof(target_id));
int ret = bake_provider_add_storage_target(
provider, target_name.c_str(), &target_id);
if(ret != 0) {
return py11::none();
}
HANDLE_ERROR(bake_provider_add_storage_target, ret);
return py11::cast(target_id);
}
static bool pybake_provider_remove_storage_target(
static void pybake_provider_remove_storage_target(
pybake_provider_t provider,
bake_target_id_t target_id)
{
return 0 == bake_provider_remove_storage_target(provider, target_id);
int ret = bake_provider_remove_storage_target(provider, target_id);
HANDLE_ERROR(bake_provider_remove_storage_target, ret);
}
static bool pybake_provider_remove_all_storage_targets(
static void pybake_provider_remove_all_storage_targets(
pybake_provider_t provider)
{
return 0 == bake_provider_remove_all_storage_targets(provider);
int ret = bake_provider_remove_all_storage_targets(provider);
HANDLE_ERROR(bake_provider_remove_all_storage_targets, ret);
}
static uint64_t pybake_provider_count_storage_targets(
pybake_provider_t provider)
{
uint64_t n = 0;
bake_provider_count_storage_targets(provider, &n);
int ret = bake_provider_count_storage_targets(provider, &n);
HANDLE_ERROR(bake_provider_count_storage_targets, ret);
return n;
}
......@@ -76,9 +87,27 @@ static py11::list pybake_provider_list_storage_targets(
return list_result;
}
static bool pybake_make_pool(const std::string& pool_name,
static void pybake_make_pool(const std::string& pool_name,
size_t pool_size, mode_t mode) {
return 0 == bake_makepool(pool_name.c_str(), pool_size, mode);
int ret = bake_makepool(pool_name.c_str(), pool_size, mode);
HANDLE_ERROR(bake_makepool, ret);
}
static void pybake_provider_set_target_xfer_buffer(
pybake_provider_t provider,
bake_target_id_t target_id,
size_t count,
size_t size) {
int ret = bake_provider_set_target_xfer_buffer(provider, target_id, count, size);
HANDLE_ERROR(bake_provider_set_target_xfer_buffer, ret);
}
static void pybake_provider_set_target_xfer_concurrency(
pybake_provider_t provider,
bake_target_id_t target_id,
uint32_t num_threads) {
int ret = bake_provider_set_target_xfer_concurrency(provider, target_id, num_threads);
HANDLE_ERROR(bake_provider_set_target_xfer_concurrency, ret);
}
PYBIND11_MODULE(_pybakeserver, m)
......@@ -91,4 +120,6 @@ PYBIND11_MODULE(_pybakeserver, m)
m.def("count_storage_targets", &pybake_provider_count_storage_targets);
m.def("list_storage_targets", &pybake_provider_list_storage_targets);
m.def("make_pool", &pybake_make_pool);
m.def("set_target_xfer_buffer", &pybake_provider_set_target_xfer_buffer);
m.def("set_target_xfer_concurrency", &pybake_provider_set_target_xfer_concurrency);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment