added client-side functions

pybake/__init__.py

 # (C) 2018 The University of Chicago
 # See COPYRIGHT in top-level directory.
-from pybake.target import BakeStorageTarget

pybake/client.py

 # (C) 2018 The University of Chicago
 # See COPYRIGHT in top-level directory.
 import _pybakeclient
+from pybake.target import *
 import pymargo
 
 class BakeClient():
+    """
+    The BakeClient class wraps a bake_client_t structure at C level.
+    It registers the RPCs necessary to interact with a Bake provider.
+    It can be used to create provider handles pointing to Bake providers.
+    """
 
-	def __init__(self, mid):
-		self._client = _pybakeclient.client_init(mid._mid)
+    def __init__(self, mid):
+        """
+        Constructor. Initializes a new BakeClient with a MargoInstance.
 
-	def create_provider_handle(self, addr, mplex_id):
-		ph = _pybakeclient.provider_handle_create(self._client, addr.get_hg_addr(), mplex_id)
-		return BakeProviderHandle(ph)
+        Args:
+            mid (MargoInstance): MargoInstance on which to register RPCs.
+        """
+        self._client = _pybakeclient.client_init(mid._mid)
 
-	def shutdown_service(self, addr):
-		_pybakeclient.shutdown_service(self._client, addr.get_hg_addr())
+    def create_provider_handle(self, addr, provider_id):
+        """
+        Creates a BakeProviderHandle object pointing to the given
+        address and provider id.
+
+        Args:
+            addr (MargoAddress): Address of the Bake provider.
+            provider_id (int): ID of the provider.
+        """
+        ph = _pybakeclient.provider_handle_create(self._client, addr.get_hg_addr(), provider_id)
+        return BakeProviderHandle(ph)
+
+    def shutdown_service(self, addr):
+        """
+        Shut down a MargoInstance running at a particular address.
+
+        Args:
+            addr (MargoAddress): Address of the MargoInstance to shut down.
+        """
+        _pybakeclient.shutdown_service(self._client, addr.get_hg_addr())
 	
-	def __del__(self):
-		_pybakeclient.client_finalize(self._client)
+    def finalize(self):
+        """
+        Finalizes the underlying bake_client_t structure.
+        """
+        _pybakeclient.client_finalize(self._client)
 
 class BakeProviderHandle():
+    """
+    The BakeProviderHandle class represents a handle to a remote Bake provider.
+    Internally, this class wraps a bake_provider_handle_t C structure.
+    """
 
-	def __init__(self, ph):
-		self._ph = ph
+    def __init__(self, ph):
+        """
+        Constructor. This is not supposed to be called by users.
+        Users should create a BakeProviderHandle from a BakeClient clt
+        by calling clt.create_provider_handle.
+        """
+        self._ph = ph
 	
-	def __del__(self):
-		_pybakeclient.provider_handle_release(self._ph)
+    def __del__(self):
+        """
+        Explicit destructor to call provider_handle_release on the underlying
+        bake_provider_handle_t C structure.
+        """
+        _pybakeclient.provider_handle_release(self._ph)
+
+    def get_eager_limit(self):
+        """
+        Get the threshold size bellow which this provider
+        handle will embed data within the RPC arguments.
+        """
+        return _pybakeclient.get_eager_limit(self._ph)
+
+    def set_eager_limit(self, limit):
+        """
+        Set the threshold size bellow which this provider handle
+        will embed data within the RPC arguments.
+        """ 
+        return _pybakeclient.set_eager_limit(self._ph, limit)
+
+    def probe(self, max_targets=0):
+        """
+        Get the list of BakeTargetIDs of targets located in
+        this provider. If max_targets is not specified, this
+        function will return all the target ids.
+        """
+        if(max_targets != 0):
+            tgts = _pybakeclient.probe(self._ph, max_targets)
+        else:
+            num_targets = 32
+            while(True):
+                tgts = _pybakeclient.probe(self._ph, num_targets)
+                if(len(tgts) == num_targets):
+                    num_targets *= 2
+                else:
+                    break
+        result = []
+        for tgt in tgts:
+            result.append(BakeTargetID(tgt))
+        return result
+
+    def create(self, bti, region_size):
+        """
+        Creates a region in the specified target, with a given size.
+
+        Args:
+            bti (BakeTargetID): ID of the bake target in which to create the region.
+            region_size (int): size of the region to create.
+
+        Returns:
+            A BakeRegionID object representing the region.
+        """
+        rid = _pybakeclient.create(self._ph, bti._tid, region_size)
+        return BakeRegionID(rid)
+
+    def write(self, rid, offset, data):
+        """
+        Writes data in a region, at a specified offset.
+
+        Args:
+            rid (BakeRegionID): region in which to write.
+            offset (int): offset at which to write.
+            data (str): data to write.
+        """
+        return _pybakeclient.write(self._ph, rid._rid, offset, data)
+
+    def persist(self, rid):
+        """
+        Make the changes to a given region persist.
+        
+        Args:
+            rid (BakeRegionID): region to persist.
+        """
+        return _pybakeclient.persist(self._ph, rid._rid)
+
+    def create_write_persist(self, bti, data):
+        """
+        Creates a new region, write data to it at a given offset,
+        and persist the region.
+
+        Args:
+            bti (BakeTargetID): target id in which to create the region.
+            size (int): size of the region to create.
+            offset (int): offset at which to write data in the region.
+            data (str): data to write.
+        """
+        rid = _pybakeclient.create_write_persist(self._ph, bti._tid, data)
+        return BakeRegionID(rid)
+
+    def get_size(self, rid):
+        """
+        Get the size of a given region.
+
+        Args:
+            rid (BakeRegionID): region id.
+
+        Returns:
+            The size (ind) of the provided region.
+        """
+        return _pybakeclient.get_size(self._ph, rid._rid)
+
+    def read(self, rid, offset=0, size=-1):
+        """
+        Reads the data contained in a given region, at a given offset
+        and with a given size. If the size is not provided,
+        this function will first send an RPC to get the current region size
+        and then read from the offset up to the end of the region. If the
+        offset is not provided, the entire region is read.
+
+        Args:
+            rid (BakeRegionID): region id.
+            offset (int): offset at which to read.
+            size (int): size to read.
+
+        Returns:
+            The data read, in the form of a string, or None if an
+            error occured.
+        """
+        if(size < 0):
+            size = self.get_size(rid) - offset
+        return _pybakeclient.read(self._ph, rid._rid, offset, size)
+    
+    def remove(self, rid):
+        """
+        Remove a region from its target.
+
+        Args:
+            rid (BakeRegionID): region to remove.
+        """
+        _pybakeclient.remove(self._ph, rid._rid)

pybake/server.py

@@ -2,7 +2,7 @@
 # See COPYRIGHT in top-level directory.
 import _pybakeserver
 import pymargo
-from pybake.target import BakeStorageTarget
+from pybake.target import BakeTargetID
 
 def make_pool(name, size, mode):
 	_pybakeserver.make_pool(name, size, mode)
@@ -15,7 +15,7 @@ class BakeProvider(pymargo.Provider):
 
 	def add_storage_target(self, name):
 		tid = _pybakeserver.add_storage_target(self._provider, name)
-		return BakeStorageTarget(tid)
+		return BakeTargetID(tid)
 
 	def remove_storage_target(self, target):
 		return _pybakeserver.remove_storage_target(self._provider, target._tid)

pybake/src/client.cpp

@@ -42,17 +42,121 @@ static bake_provider_handle_t pybake_provider_handle_create(
     return providerHandle;
 }
 
+static uint64_t pybake_get_eager_limit(
+        bake_provider_handle_t ph)
+{
+    uint64_t limit;
+    int ret = bake_provider_handle_get_eager_limit(ph, &limit);
+    if(ret != 0) return 0;
+    return limit;
+}
+
+static bpl::object pybake_probe(
+        bake_provider_handle_t ph,
+        uint64_t max_targets)
+{
+    bpl::list result;
+    std::vector<bake_target_id_t> targets(max_targets);
+    uint64_t num_targets;
+    int ret = bake_probe(ph, max_targets, targets.data(), &num_targets);
+    if(ret != 0) return bpl::object();
+    for(uint64_t i=0; i < num_targets; i++) {
+        result.append(bpl::object(targets[i]));
+    }
+    return result;
+}
+
+static bpl::object pybake_create(
+        bake_provider_handle_t ph,
+        bake_target_id_t bti,
+        size_t region_size)
+{
+    bake_region_id_t rid;
+    std::memset(&rid, 0, sizeof(rid));
+    int ret = bake_create(ph, bti, region_size, &rid);
+    if(ret != 0) return bpl::object();
+    else return bpl::object(rid);
+}
+
+static bpl::object pybake_write(
+        bake_provider_handle_t ph,
+        const bake_region_id_t& rid,
+        uint64_t offset,
+        const std::string& data)
+{
+    int ret = bake_write(ph, rid, offset, (const void*)data.data(), data.size());
+    if(ret == 0) return bpl::object(true);
+    else return bpl::object(false);
+}
+
+static bpl::object pybake_persist(
+        bake_provider_handle_t ph,
+        const bake_region_id_t& rid)
+{
+    int ret = bake_persist(ph, rid);
+    if(ret == 0) return bpl::object(true);
+    else return bpl::object(false);
+}
+
+static bpl::object pybake_create_write_persist(
+        bake_provider_handle_t ph,
+        bake_target_id_t tid,
+        const std::string& data)
+{
+    bake_region_id_t rid;
+    int ret = bake_create_write_persist(ph, tid, 
+            data.data(), data.size(), &rid);
+    if(ret == 0) return bpl::object(rid);
+    else return bpl::object();
+}
+
+static bpl::object pybake_get_size(
+        bake_provider_handle_t ph,
+        const bake_region_id_t& rid)
+{
+    uint64_t size;
+    int ret = bake_get_size(ph, rid, &size);
+    if(ret == 0) return bpl::object(size);
+    else return bpl::object();
+}
+
+static bpl::object pybake_read(
+        bake_provider_handle_t ph,
+        const bake_region_id_t& rid,
+        uint64_t offset,
+        size_t size) 
+{
+    std::string result(size, '\0');
+    uint64_t bytes_read;
+    int ret = bake_read(ph, rid, offset, (void*)result.data(), size, &bytes_read);
+    if(ret != 0) return bpl::object();
+    result.resize(bytes_read);
+    return bpl::object(result);
+}
+
 BOOST_PYTHON_MODULE(_pybakeclient)
 {
 #define ret_policy_opaque bpl::return_value_policy<bpl::return_opaque_pointer>()
 
+    bpl::import("_pybaketarget");
     bpl::opaque<bake_client>();
     bpl::opaque<bake_provider_handle>();
+//    bpl::class_<bake_region_id_t>("bake_region_id", bpl::no_init);
     bpl::def("client_init", &pybake_client_init, ret_policy_opaque);
     bpl::def("client_finalize", &bake_client_finalize);
     bpl::def("provider_handle_create", &pybake_provider_handle_create, ret_policy_opaque);
     bpl::def("provider_handle_ref_incr", &bake_provider_handle_ref_incr);
     bpl::def("provider_handle_release", &bake_provider_handle_release);
+    bpl::def("get_eager_limit", &pybake_get_eager_limit);
+    bpl::def("set_eager_limit", &bake_provider_handle_set_eager_limit);
+    bpl::def("probe", &pybake_probe);
+    bpl::def("create", &pybake_create);
+    bpl::def("write", &pybake_write);
+    bpl::def("persist", &pybake_persist);
+    bpl::def("create_write_persist", &pybake_create_write_persist);
+    bpl::def("get_size", &pybake_get_size);
+    bpl::def("read", &pybake_read);
+    bpl::def("remove", &bake_remove);
     bpl::def("shutdown_service", &bake_shutdown_service);
 
 #undef ret_policy_opaque

pybake/src/server.cpp

@@ -40,13 +40,13 @@ static bake_target_id_t pybake_provider_add_storage_target(
                 provider, target_name.c_str(), &target_id);
     return target_id;
 }
-
+#if 0
 static std::string pybake_target_id_to_string(bake_target_id_t tid) {
     char id[37];
     uuid_unparse(tid.id, id);
     return std::string(id);
 }
-
+#endif
 static bool pybake_provider_remove_storage_target(
         bake_provider_t provider,
         bake_target_id_t target_id)
@@ -90,10 +90,10 @@ BOOST_PYTHON_MODULE(_pybakeserver)
 {
 #define ret_policy_opaque bpl::return_value_policy<bpl::return_opaque_pointer>()
 
+    bpl::import("_pybaketarget");
     bpl::opaque<bake_server_context_t>();
-    bpl::class_<bake_target_id_t>("bake_target_id", bpl::no_init)
-        .def("__str__", pybake_target_id_to_string);
-    bpl::class_<bake_region_id_t>("bake_region_id", bpl::no_init);
+//    bpl::class_<bake_target_id_t>("bake_target_id", bpl::no_init)
+//        .def("__str__", pybake_target_id_to_string);
     bpl::def("register", &pybake_provider_register, ret_policy_opaque);
     bpl::def("add_storage_target", &pybake_provider_add_storage_target);
     bpl::def("remove_storage_target", &pybake_provider_remove_storage_target);

pybake/src/target.cpp

+/*
+ * (C) 2018 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
+#define BOOST_NO_AUTO_PTR
+#include <boost/python.hpp>
+#include <boost/python/return_opaque_pointer.hpp>
+#include <boost/python/handle.hpp>
+#include <boost/python/enum.hpp>
+#include <boost/python/def.hpp>
+#include <boost/python/module.hpp>
+#include <boost/python/return_value_policy.hpp>
+#include <string>
+#include <vector>
+#include <cstring>
+#include <iostream>
+#include <margo.h>
+#include <bake.h>
+#include <bake-server.h>
+
+namespace bpl = boost::python;
+
+static std::string pybake_target_id_to_string(bake_target_id_t tid) {
+    char id[37];
+    uuid_unparse(tid.id, id);
+    return std::string(id);
+}
+
+static bpl::object pybake_target_id_from_string(const std::string& tidstr) {
+    bake_target_id_t tid;
+    memset(tid.id, 0, sizeof(uuid_t));
+    if(tidstr.size() != 36) return bpl::object();
+    int ret = uuid_parse((char*)tidstr.c_str(), tid.id);
+    if(ret == 0) return bpl::object(tid);
+    else return bpl::object();
+}
+
+static std::string pybake_region_id_to_string(const bake_region_id_t& region_id) {
+    std::string result((const char*)(&region_id), sizeof(region_id));
+    return result;
+}
+
+static bpl::object pybake_region_id_from_string(const std::string& region_str) {
+    bake_region_id_t result;
+    memset(&result, 0, sizeof(result));
+    if(region_str.size() != sizeof(bake_region_id_t))
+        return bpl::object();
+    memcpy(&result, region_str.data(), sizeof(bake_region_id_t));
+    return bpl::object(result);
+}
+
+BOOST_PYTHON_MODULE(_pybaketarget)
+{
+    bpl::class_<bake_target_id_t>("bake_target_id", bpl::no_init)
+        .def("__str__", pybake_target_id_to_string);
+    bpl::def("target_id_from_string", pybake_target_id_from_string);
+    bpl::class_<bake_region_id_t>("bake_region_id", bpl::no_init)
+        .def("__str__", pybake_region_id_to_string);
+    bpl::def("region_id_from_string", pybake_region_id_from_string);    
+}

pybake/target.py

 # (C) 2018 The University of Chicago
 # See COPYRIGHT in top-level directory.
-class BakeStorageTarget():
+import _pybaketarget
+import base64
 
-	def __init__(self, tid):
-		self.__tid = tid
+class BakeTargetID():
 
-	def __str__(self):
-		return str(self.__tid)
+    def __init__(self, tid):
+        self._tid = tid
+
+    def __str__(self):
+        return str(self._tid)
+
+    @staticmethod
+    def from_str(string):
+        tid = _pybaketarget.target_id_from_string(string)
+        if(tid is None):
+            return None
+        else:
+            return BakeTargetID(tid)
+
+class BakeRegionID():
+
+    def __init__(self, rid):
+        self._rid = rid
+    
+    def __str__(self):
+        return base64.b64encode(str(self._rid))
+
+    @staticmethod
+    def from_str(string):
+        rid = _pybaketarget.region_id_from_string(base64.b64decode(string))
+        if(rid is None):
+            return None
+        else:
+            return BakeRegionID(rid)

setup.py

@@ -12,17 +12,23 @@ os.environ['OPT'] = " ".join(
 
 pybake_server_module = Extension('_pybakeserver', ["pybake/src/server.cpp"],
 		           libraries=['boost_python','margo','bake-server'],
-			   include_dirs=['.'],
-			   depends=[])
+                   include_dirs=['.'],
+                   depends=[])
 
 pybake_client_module = Extension('_pybakeclient', ["pybake/src/client.cpp"],
 		           libraries=['boost_python','margo','bake-client'],
-			   include_dirs=['.'],
-			   depends=[])
+                   include_dirs=['.'],
+                   depends=[])
+
+pybake_target_module = Extension('_pybaketarget', ["pybake/src/target.cpp"],
+                    libraries=['boost_python', 'uuid' ],
+                    include_dirs=['.'],
+                    depends=[])
+
 setup(name='pybake',
       version='0.1',
       author='Matthieu Dorier',
       description="""Python binding for BAKE""",      
-      ext_modules=[ pybake_server_module, pybake_client_module ],
+      ext_modules=[ pybake_server_module, pybake_client_module, pybake_target_module ],
       packages=['pybake']
      )

test/client.py

+# (C) 2018 The University of Chicago
+# See COPYRIGHT in top-level directory.
+import sys
+from pymargo import MargoInstance
+from pybake.target import BakeRegionID
+from pybake.client import *
+
+mid = MargoInstance('tcp')
+
+server_addr = sys.argv[1]
+mplex_id    = int(sys.argv[2])
+
+client = BakeClient(mid)
+addr = mid.lookup(server_addr)
+ph = client.create_provider_handle(addr, mplex_id)
+
+# Testing get_eager_limit
+lim = ph.get_eager_limit()
+print "Eager limit is: "+str(lim)
+
+# probe the provider handle (for all targets)
+targets = ph.probe()
+print "Probe found the following targets:"
+for t in targets:
+    print "===== "+str(t)
+
+target = targets[0]
+# create a 32-bytes region in the first target
+region = ph.create(target, 32)
+print "Created region "+str(region)
+
+regionstr = str(region)
+region = BakeRegionID.from_str(regionstr)
+
+# write into the region
+ph.write(region, 0, 'A'*16)
+ph.write(region, 16, 'B'*16)
+
+# get size of region
+s = ph.get_size(region)
+print "Region size is "+str(s)
+
+# persist region
+ph.persist(region)
+
+# read region
+result = ph.read(region, 8, 16)
+print "Reading region at offset 8, size 16 gives: "+str(result)
+
+del ph
+client.shutdown_service(addr)
+del addr
+client.finalize()
+mid.finalize()

test/server.py

@@ -25,7 +25,7 @@ targets = provider.list_storage_targets()
 for t in targets:
   print str(t)
 
-provider.remove_all_storage_targets()
-print "number of targets: "+str(provider.count_storage_targets())
+#provider.remove_all_storage_targets()
+#print "number of targets: "+str(provider.count_storage_targets())
 
 mid.wait_for_finalize()