added proxy functions and more comments

pybake/client.py

@@ -80,7 +80,7 @@ class BakeProviderHandle():
         Set the threshold size bellow which this provider handle
         will embed data within the RPC arguments.
         """ 
-        return _pybakeclient.set_eager_limit(self._ph, limit)
+        _pybakeclient.set_eager_limit(self._ph, limit)
 
     def probe(self, max_targets=0):
         """
@@ -115,8 +115,6 @@ class BakeProviderHandle():
             A BakeRegionID object representing the region. None if an error occured.
         """
         rid = _pybakeclient.create(self._ph, bti._tid, region_size)
-        if(rid == None):
-            return None
         return BakeRegionID(rid)
 
     def write(self, rid, offset, data):
@@ -127,10 +125,8 @@ class BakeProviderHandle():
             rid (BakeRegionID): region in which to write.
             offset (int): offset at which to write.
             data (str): data to write.
-        Returns:
-            True if the data was correctly written, False otherwise.
         """
-        return _pybakeclient.write(self._ph, rid._rid, offset, data)
+        _pybakeclient.write(self._ph, rid._rid, offset, data)
 
     def write_numpy(self, rid, offset, array):
         """
@@ -140,21 +136,39 @@ class BakeProviderHandle():
             rid (BakeRegionID): region in which to write.
             offset (int): offset at which to write.
             data (numpy.ndarray): numpy array to write.
-        Returns:
-            True if the data was correctly written, False otherwise.
         """
-        return _pybakeclient.write_numpy(self._ph, rid._rid, offset, array)
+        _pybakeclient.write_numpy(self._ph, rid._rid, offset, array)
+
+    def proxy_write(self, rid, bulk, size, offset_in_region=0, offset_in_bulk=0, remote_addr=''):
+        """
+        Writes data that is already exposed in a pymargo.bulk.Bulk object.
+
+        Args:
+            rid (BakeRegionID): region in which to write.
+            bulk (Bulk): bulk handle from which to get the data.
+            size (int): size to write.
+            offset_in_region (int): offset at which to write in the region.
+            offset_in_bulk (int): offset from which to read in the bulk object.
+            remote_addr (str): address of the process that created the bulk object.
+        """
+        _pybakeclient.proxy_write(self._ph, rid._rid, offset_in_region,
+                bulk._hg_bulk, offset_in_bulk, remote_addr, size)
 
-    def persist(self, rid):
+    def persist(self, rid, offset=0, size=-1):
         """
         Make the changes to a given region persist.
+        Offset is 0 by default. If size is not provided, PyBake will
+        call get_size to get it, which will throw an exception if Bake
+        has not been compiled with --enable-sizecheck.
         
         Args:
             rid (BakeRegionID): region to persist.
-        Returns:
-            True if the region was correctly persisted, False otherwise.
+            offset (int): offset in the region.
+            size (int): number of bytes to persist.
         """
-        return _pybakeclient.persist(self._ph, rid._rid)
+        if(size < 0):
+            size = self.get_size(rid) - offset
+        _pybakeclient.persist(self._ph, rid._rid, offset, size)
 
     def create_write_persist(self, bti, data):
         """
@@ -167,11 +181,27 @@ class BakeProviderHandle():
             offset (int): offset at which to write data in the region.
             data (str): data to write.
         Returns:
-            True if the region was correctly written and persisted, False otherwise.
+            The created BakeRegionID.
         """
         rid = _pybakeclient.create_write_persist(self._ph, bti._tid, data)
-        if(rid is None):
-            return None
+        return BakeRegionID(rid)
+
+    def proxy_create_write_persist(self, bti, bulk, size, offset_in_bulk=0, remote_addr=''):
+        """
+        Version of create_write_persist that takes an already created bulk handle
+        as argument as well as the address of the process that created it.
+
+        Args:
+            bti (BakeTargetID): target id in which to create the region.
+            bulk (Bulk): bulk handle.
+            size (int): size of the region to create.
+            offset_in_bulk (int): offset in the bulk data.
+            remote_addr (str): Address of the process that created the bulk.
+        Returns:
+            The created BakeRegionID.
+        """
+        rid = _pybakeclient.create_write_persist_proxy(self._ph,
+                bti._tid, bulk._hg_bulk, offset_in_bulk, remote_addr, size)
         return BakeRegionID(rid)
 
     def create_write_persist_numpy(self, bti, array):
@@ -185,22 +215,21 @@ class BakeProviderHandle():
             offset (int): offset at which to write data in the region.
             array (numpy.ndarray): numpy array to write.
         Returns:
-            True if the region was correctly written and persisted, False otherwise.
+            The created BakeRegionID.
         """
         rid = _pybakeclient.create_write_persist_numpy(self._ph, bti._tid, array)
-        if(rid is None):
-            return None
         return BakeRegionID(rid)
 
     def get_size(self, rid):
         """
-        Get the size of a given region.
+        Get the size of a given region. Note thate bake should have been compiler
+        with --enable-sizecheck, otherwise this function will throw an exception.
 
         Args:
             rid (BakeRegionID): region id.
 
         Returns:
-            The size (int) of the provided region. None in case of error.
+            The size (int) of the provided region.
         """
         return _pybakeclient.get_size(self._ph, rid._rid)
 
@@ -212,6 +241,10 @@ class BakeProviderHandle():
         and then read from the offset up to the end of the region. If the
         offset is not provided, the entire region is read.
 
+        Note that if bake has not been compiled with --enable-sizecheck,
+        then not providing the size argument will make this function throw
+        an exception.
+
         Args:
             rid (BakeRegionID): region id.
             offset (int): offset at which to read.
@@ -224,7 +257,24 @@ class BakeProviderHandle():
         if(size < 0):
             size = self.get_size(rid) - offset
         return _pybakeclient.read(self._ph, rid._rid, offset, size)
-    
+   
+    def proxy_read(self, rid, bulk, size, offset_in_region=0, offset_in_bulk=0, remote_addr=''):
+        """
+        Reads the data contained in a given region and pushes it to a provided bulk handle.
+
+        Args:
+            rid (BakeRegionID): region id.
+            bulk (Bulk): bulk handle where to push the data.
+            size (int): size to read.
+            offset_in_region (int): offset in the region where to start reading.
+            offset_in_bulk (int): offset in the bulk where to start placing data.
+            remote_addr (str): address of the process that created the Bulk object.
+        Returns: 
+            the effective number of bytes read.
+        """
+        return _pybakeclient.proxy_read(self._ph, rid._rid, offset_in_region, bulk._hg_bulk,
+                offset_in_bulk, remote_addr, size)
+
     def read_numpy(self, rid, offset, shape, dtype):
         """
         Reads the data contained in a given region, at a given offset,
@@ -251,8 +301,7 @@ class BakeProviderHandle():
         Args:
             rid (BakeRegionID): region to remove.
         """
-        ret = _pybakeclient.remove(self._ph, rid._rid)
-        return (ret == 0)
+        _pybakeclient.remove(self._ph, rid._rid)
 
     def migrate_region(self, source_rid, dest_addr, dest_provider_id, dest_target, remove_source=True):
         """
@@ -271,8 +320,6 @@ class BakeProviderHandle():
         """
         ret = _pybakeclient.migrate_region(self._ph, source_rid._rid, remove_source,
                 str(dest_addr), int(dest_provider_id), dest_target._tid)
-        if(ret == None):
-            return ret
         return BakeRegionID(ret)
 
     def migrate_target(self, source_tid, dest_addr, dest_provider_id, dest_root, remove_source=True):
@@ -290,5 +337,5 @@ class BakeProviderHandle():
         Returns:
             True if the target was correctly migrated.
         """
-        return _pybakeclient.migrate_target(self._ph, source_tid._tid, remove_source,
+        _pybakeclient.migrate_target(self._ph, source_tid._tid, remove_source,
                 str(dest_addr), int(dest_provider_id), str(dest_root))

pybake/server.py

@@ -9,27 +9,63 @@ def make_pool(name, size, mode):
     _pybakeserver.make_pool(name, size, mode)
 
 class BakeProvider(pymargo.Provider):
+    """
+    The BakeProvide class wraps a C-level bake_provider_t object.
+    """
 
-    def __init__(self, mid, provider_id):
-        super(BakeProvider, self).__init__(mid, provider_id)
+    def __init__(self, engine, provider_id):
+        """
+        Constructor. Initializes a provider with an Engine and provider_id.
+        """
+        super(BakeProvider, self).__init__(engine, provider_id)
         self._provider = _pybakeserver.register(mid._mid, provider_id)
 
-    def add_storage_target(self, name):
-        tid = _pybakeserver.add_storage_target(self._provider, name)
+    def add_storage_target(self, path):
+        """
+        Adds a storage target to the provider.
+        Returns a BakeTargetID instance that can be used to access the storage target.
+        """
+        tid = _pybakeserver.add_storage_target(self._provider, path)
         return BakeTargetID(tid)
 
     def remove_storage_target(self, target):
-        return _pybakeserver.remove_storage_target(self._provider, target._tid)
+        """
+        Removes a storage target from the provider. This does not delete the underlying file.
+        The target argument must be a BakeTargetID object.
+        """
+        _pybakeserver.remove_storage_target(self._provider, target._tid)
 
     def remove_all_storage_targets(self):
-        return _pybakeserver.remove_all_storage_targets(self._provider)
+        """
+        Removes all the storage targets managed by this provider.
+        """
+        _pybakeserver.remove_all_storage_targets(self._provider)
 
     def count_storage_targets(self):
+        """
+        Returns the number of storage targets that this provider manages.
+        """
         return _pybakeserver.count_storage_targets(self._provider)
 
     def list_storage_targets(self):
+        """
+        Returns the list of storage targets (BakeTargetIDs) that this provider manages.
+        """
         l = _pybakeserver.list_storage_targets(self._provider)
         if(l is None):
             return []
         else:
             return [ BakeTargetID(tid) for tid in l ]
+
+    def set_target_xfer_buffer(self, target, count, size):
+        """
+        Sets the number and size of intermediate buffers that can be used to
+        execute transfers to a specific target.
+        """
+        _pybakeserver.set_target_xfer_buffer(self._provider, target._tid, count, size)
+
+    def set_target_xfer_concurrency(self, target, num_threads):
+        """
+        Sets the number of ULTs that can be used to execute transfers concurrently.
+        """
+        _pybakeserver.set_target_xfer_concurrency(self._provider, target._tid, num_threads)

pybake/src/server.cpp

@@ -4,6 +4,8 @@
  * See COPYRIGHT in top-level directory.
  */
 #include <pybind11/pybind11.h>
+#include <sstream>
+#include <stdexcept>
 #include <string>
 #include <vector>
 #include <cstring>
@@ -14,6 +16,14 @@
 
 namespace py11 = pybind11;
 
+#define HANDLE_ERROR(__func, __ret) do {\
+    if(__ret != BAKE_SUCCESS) {\
+        std::stringstream ss;\
+        ss << #__func << "() failed (ret = " << __ret << ")";\
+        throw std::runtime_error(ss.str());\
+    }\
+} while(0)
+
 typedef py11::capsule pymargo_instance_id;
 typedef py11::capsule pymargo_addr;
 typedef py11::capsule pybake_provider_t;
@@ -25,8 +35,8 @@ typedef py11::capsule pybake_provider_t;
 static pybake_provider_t pybake_provider_register(pymargo_instance_id mid, uint8_t provider_id) {
     bake_provider_t provider;
     int ret = bake_provider_register(mid, provider_id, BAKE_ABT_POOL_DEFAULT, &provider);
-    if(ret != 0) return py11::none();
-    else return BAKEPR2CAPSULE(provider);
+    HANDLE_ERROR(bake_provider_register, ret);
+    return BAKEPR2CAPSULE(provider);
 }
 
 static py11::object pybake_provider_add_storage_target(
@@ -36,30 +46,31 @@ static py11::object pybake_provider_add_storage_target(
     std::memset(&target_id, 0, sizeof(target_id));
     int ret = bake_provider_add_storage_target(
                 provider, target_name.c_str(), &target_id);
-    if(ret != 0) {
-        return py11::none();
-    }
+    HANDLE_ERROR(bake_provider_add_storage_target, ret);
     return py11::cast(target_id);
 }
 
-static bool pybake_provider_remove_storage_target(
+static void pybake_provider_remove_storage_target(
         pybake_provider_t provider,
         bake_target_id_t target_id)
 {
-    return 0 == bake_provider_remove_storage_target(provider, target_id);
+    int ret = bake_provider_remove_storage_target(provider, target_id);
+    HANDLE_ERROR(bake_provider_remove_storage_target, ret);
 }
 
-static bool pybake_provider_remove_all_storage_targets(
+static void pybake_provider_remove_all_storage_targets(
         pybake_provider_t provider)
 {
-    return 0 == bake_provider_remove_all_storage_targets(provider);
+    int ret = bake_provider_remove_all_storage_targets(provider);
+    HANDLE_ERROR(bake_provider_remove_all_storage_targets, ret);
 }
 
 static uint64_t pybake_provider_count_storage_targets(
         pybake_provider_t provider)
 {
     uint64_t n = 0;
-    bake_provider_count_storage_targets(provider, &n);
+    int ret = bake_provider_count_storage_targets(provider, &n);
+    HANDLE_ERROR(bake_provider_count_storage_targets, ret);
     return n;
 }
 
@@ -76,9 +87,27 @@ static py11::list pybake_provider_list_storage_targets(
     return list_result;
 }
 
-static bool pybake_make_pool(const std::string& pool_name,
+static void pybake_make_pool(const std::string& pool_name,
                 size_t pool_size, mode_t mode) {
-    return 0 == bake_makepool(pool_name.c_str(), pool_size, mode);
+    int ret = bake_makepool(pool_name.c_str(), pool_size, mode);
+    HANDLE_ERROR(bake_makepool, ret);
+}
+
+static void pybake_provider_set_target_xfer_buffer(
+        pybake_provider_t provider,
+        bake_target_id_t target_id,
+        size_t count,
+        size_t size) {
+    int ret = bake_provider_set_target_xfer_buffer(provider, target_id, count, size);
+    HANDLE_ERROR(bake_provider_set_target_xfer_buffer, ret);
+}
+
+static void pybake_provider_set_target_xfer_concurrency(
+        pybake_provider_t provider,
+        bake_target_id_t target_id,
+        uint32_t num_threads) {
+    int ret = bake_provider_set_target_xfer_concurrency(provider, target_id, num_threads);
+    HANDLE_ERROR(bake_provider_set_target_xfer_concurrency, ret);
 }
 
 PYBIND11_MODULE(_pybakeserver, m)
@@ -91,4 +120,6 @@ PYBIND11_MODULE(_pybakeserver, m)
     m.def("count_storage_targets", &pybake_provider_count_storage_targets);
     m.def("list_storage_targets", &pybake_provider_list_storage_targets);
     m.def("make_pool", &pybake_make_pool);
+    m.def("set_target_xfer_buffer", &pybake_provider_set_target_xfer_buffer);
+    m.def("set_target_xfer_concurrency", &pybake_provider_set_target_xfer_concurrency);
 }