WriteBatch working for datasets

include/hepnos.hpp

@@ -13,5 +13,6 @@
 #include <hepnos/RunSet.hpp>
 #include <hepnos/SubRun.hpp>
 #include <hepnos/SubRunNumber.hpp>
+#include <hepnos/WriteBatch.hpp>
 
 #endif

include/hepnos/DataSet.hpp

@@ -161,7 +161,10 @@ class DataSet : public KeyValueContainer {
      * @return a valid ProductID if the key did not already exist and the write succeeded,
      *      an invalid one otherwise.
      */
-    ProductID storeRawData(const std::string& key, const std::vector<char>& buffer) override;
+    ProductID storeRawData(const std::string& key, const std::string& value) override;
+    ProductID storeRawData(std::string&& key, std::string&& value) override;
+    ProductID storeRawData(WriteBatch& batch, const std::string& key, const std::string& value) override;
+    ProductID storeRawData(WriteBatch& batch, std::string&& key, std::string&& value) override;
 
     /**
      * @brief Loads binary data associated with a particular key from the DataSet.
@@ -174,7 +177,7 @@ class DataSet : public KeyValueContainer {
      * @return true if the key exists and the read succeeded,
      *      false otherwise.
      */
-    bool loadRawData(const std::string& key, std::vector<char>& buffer) const override;
+    bool loadRawData(const std::string& key, std::string& value) const override;
 
     /**
      * @brief Comparison operator.

include/hepnos/DataStore.hpp

@@ -19,6 +19,7 @@ class Run;
 class SubRun;
 class Event;
 template<typename T, typename C = std::vector<T>> class Ptr;
+class WriteBatch;
 
 /**
  * The DataStore class is the main handle referencing an HEPnOS service.
@@ -32,6 +33,7 @@ class DataStore {
     friend class Run;
     friend class SubRun;
     friend class Event;
+    friend class WriteBatch;
 
     public:
 
@@ -249,6 +251,18 @@ class DataStore {
      */
     DataSet createDataSet(const std::string& name);
 
+    /**
+     * @brief Creates a dataset with a given name inside the data store.
+     * This function takes a WriteBatch instance, the dataset will be
+     * actually created when this batch is flushed or destroyed.
+     *
+     * @param batch WriteBatch in which to enqueue the creation.
+     * @param name Name of the dataset.
+     *
+     * @return A DataSet instance pointing to the created dataset.
+     */
+    DataSet createDataSet(WriteBatch& batch, const std::string& name);
+
     /**
      * @brief Create a pointer to a product. The type T used must
      * match the type of the product corresponding to the provided
@@ -315,7 +329,7 @@ class DataStore {
      *
      * @return true if the data was loaded successfuly, false otherwise.
      */
-    bool loadRawProduct(const ProductID& productID, std::vector<char>& buffer);
+    bool loadRawProduct(const ProductID& productID, std::string& buffer);
 };
 
 class DataStore::const_iterator {
@@ -560,7 +574,7 @@ Ptr<T,C> DataStore::makePtr(const ProductID& productID, std::size_t index) {
 
 template<typename T>
 bool DataStore::loadProduct(const ProductID& productID, T& t) {
-    std::vector<char> buffer;
+    std::string buffer;
     if(!loadRawProduct(productID, buffer)) {
         return false;
     }

include/hepnos/Event.hpp

@@ -105,21 +105,24 @@ class Event : public KeyValueContainer {
      * @brief Stores raw key/value data in this Event.
      *
      * @param key Key
-     * @param buffer Value
+     * @param value Value
      *
      * @return a valid ProductID if the key did not already exist, an invalid one otherwise.
      */
-    ProductID storeRawData(const std::string& key, const std::vector<char>& buffer) override;
+    ProductID storeRawData(const std::string& key, const std::string& value) override;
+    ProductID storeRawData(std::string&& key, std::string&& value) override;
+    ProductID storeRawData(WriteBatch& batch, const std::string& key, const std::string& value) override;
+    ProductID storeRawData(WriteBatch& batch, std::string&& key, std::string&& value) override;
 
     /**
      * @brief Loads raw key/value data from this Event.
      *
      * @param key Key
-     * @param buffer Buffer used to hold the value.
+     * @param value Buffer used to hold the value.
      *
      * @return true if the key exists, false otherwise.
      */
-    bool loadRawData(const std::string& key, std::vector<char>& buffer) const override;
+    bool loadRawData(const std::string& key, std::string& value) const override;
 
     /**
      * @brief Compares this Event with another Event. The Events must point to

include/hepnos/KeyValueContainer.hpp

@@ -19,6 +19,8 @@
 
 namespace hepnos {
 
+class WriteBatch;
+
 class KeyValueContainer {
 
     public:
@@ -69,7 +71,13 @@ class KeyValueContainer {
      *
      * @return A valid ProductID if the key did not already exist, an invalid one otherwise.
      */
-    virtual ProductID storeRawData(const std::string& key, const std::vector<char>& buffer) = 0;
+    virtual ProductID storeRawData(const std::string& key, const std::string& value) = 0;
+
+    virtual ProductID storeRawData(std::string&& key, std::string&& value) = 0;
+
+    virtual ProductID storeRawData(WriteBatch& batch, const std::string& key, const std::string& value) = 0;
+
+    virtual ProductID storeRawData(WriteBatch& batch, std::string&& key, std::string&& value) = 0;
 
     /**
      * @brief Loads raw key/value data from this KeyValueContainer.
@@ -80,7 +88,7 @@ class KeyValueContainer {
      *
      * @return true if the key exists, false otherwise.
      */
-    virtual bool loadRawData(const std::string& key, std::vector<char>& buffer) const = 0;
+    virtual bool loadRawData(const std::string& key, std::string& buffer) const = 0;
 
     /**
      * @brief Stores a key/value pair into the KeyValueContainer.
@@ -99,18 +107,16 @@ class KeyValueContainer {
      */
     template<typename K, typename V>
     ProductID store(const K& key, const V& value) {
-        std::stringstream ss_value;
-        std::stringstream ss_key;
-        ss_key << key << "#" << demangle<V>();
-        boost::archive::binary_oarchive oa(ss_value);
-        try {
-            oa << value;
-        } catch(...) {
-            throw Exception("Exception occured during serialization");
-        }
-        std::string serialized = ss_value.str();
-        std::vector<char> buffer(serialized.begin(), serialized.end());
-        return storeRawData(ss_key.str(), buffer);
+        std::string key_str, val_str;
+        serializeKeyValue(key, value, key_str, val_str);
+        return storeRawData(std::move(key_str), std::move(val_str));
+    }
+
+    template<typename K, typename V>
+    ProductID store(WriteBatch& batch, const K& key, const V& value) {
+        std::string key_str, val_str;
+        serializeKeyValue(key, value, key_str, val_str);
+        return storeRawData(batch, std::move(key_str), std::move(val_str));
     }
 
     /**
@@ -130,15 +136,14 @@ class KeyValueContainer {
      */
     template<typename K, typename V>
     bool load(const K& key, V& value) const {
-        std::vector<char> buffer;
+        std::string buffer;
         std::stringstream ss_key;
         ss_key << key << "#" << demangle<V>();
         if(!loadRawData(ss_key.str(), buffer)) {
             return false;
         }
         try {
-            std::string serialized(buffer.begin(), buffer.end());
-            std::stringstream ss(serialized);
+            std::stringstream ss(buffer);
             //boost::archive::binary_iarchive ia(ss);
             InputArchive ia(getDataStore(), ss);
             ia >> value;
@@ -147,6 +152,24 @@ class KeyValueContainer {
         }
         return true;
     }
+
+    private:
+
+    template<typename K, typename V>
+    static void serializeKeyValue(const K& key, const V& value,
+            std::string& key_str, std::string& value_str) {
+        std::stringstream ss_value;
+        std::stringstream ss_key;
+        ss_key << key << "#" << demangle<V>();
+        key_str = std::move(ss_key.str());
+        boost::archive::binary_oarchive oa(ss_value);
+        try {
+            oa << value;
+        } catch(...) {
+            throw Exception("Exception occured during serialization");
+        }
+        value_str = ss_value.str();
+    }
 };
 
 }

include/hepnos/ProductID.hpp

@@ -11,14 +11,16 @@
 #include <boost/serialization/access.hpp>
 #include <boost/serialization/string.hpp>
 
-#include <hepnos/DataStore.hpp>
 
 namespace hepnos {
 
+class WriteBatch;
+class DataStore;
+
 class ProductID {
 
     friend class DataStore;
-    friend class DataStore::Impl;
+    friend class WriteBatch;
     friend class boost::serialization::access;
 
     private:

include/hepnos/Run.hpp

@@ -115,8 +115,10 @@ class Run : public KeyValueContainer {
      *
      * @return a valid ProductID if the key did not already exist, an invalid one otherwise.
      */
-    ProductID storeRawData(const std::string& key, const std::vector<char>& buffer) override;
-
+    ProductID storeRawData(const std::string& key, const std::string& value) override;
+    ProductID storeRawData(std::string&& key, std::string&& value) override;
+    ProductID storeRawData(WriteBatch& batch, const std::string& key, const std::string& value) override;
+    ProductID storeRawData(WriteBatch& batch, std::string&& key, std::string&& value) override;
     /**
      * @brief Loads raw key/value data from this Run.
      *
@@ -125,7 +127,7 @@ class Run : public KeyValueContainer {
      *
      * @return true if the key exists, false otherwise.
      */
-    bool loadRawData(const std::string& key, std::vector<char>& buffer) const override;
+    bool loadRawData(const std::string& key, std::string& buffer) const override;
 
     /**
      * @brief Compares this Run with another Run. The Runs must point to

include/hepnos/SubRun.hpp

@@ -112,7 +112,10 @@ class SubRun : public KeyValueContainer {
      *
      * @return a valid ProductID if the key did not already exist, an invalid one otherwise.
      */
-    ProductID storeRawData(const std::string& key, const std::vector<char>& buffer) override;
+    ProductID storeRawData(const std::string& key, const std::string& value) override;
+    ProductID storeRawData(std::string&& key, std::string&& value) override;
+    ProductID storeRawData(WriteBatch& batch, const std::string& key, const std::string& value) override;
+    ProductID storeRawData(WriteBatch& batch, std::string&& key, std::string&& value) override;
 
     /**
      * @brief Loads raw key/value data from this SubRun.
@@ -122,7 +125,7 @@ class SubRun : public KeyValueContainer {
      *
      * @return true if the key exists, false otherwise.
      */
-    bool loadRawData(const std::string& key, std::vector<char>& buffer) const override;
+    bool loadRawData(const std::string& key, std::string& buffer) const override;
 
     /**
      * @brief Compares this SubRun with another SubRun. The SubRuns must point to

include/hepnos/WriteBatch.hpp

+/*
+ * (C) 2019 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
+#ifndef __HEPNOS_WRITE_BATCH_H
+#define __HEPNOS_WRITE_BATCH_H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <hepnos/KeyValueContainer.hpp>
+#include <hepnos/ProductID.hpp>
+#include <hepnos/DataStore.hpp>
+#include <hepnos/Exception.hpp>
+
+namespace hepnos {
+
+class DataStore;
+class DataSet;
+class Run;
+class SubRun;
+class Event;
+
+class WriteBatch {
+
+    friend class DataStore;
+    friend class DataSet;
+    friend class Run;
+    friend class SubRun;
+    friend class Event;
+    friend class KeyValueContainer;
+
+    private:
+
+    class Impl;
+
+    std::unique_ptr<Impl> m_impl;
+
+    public:
+
+    WriteBatch(DataStore& ds);
+    ~WriteBatch();
+    WriteBatch(const WriteBatch&) = delete;
+    WriteBatch& operator=(const WriteBatch&) = delete;
+    WriteBatch(WriteBatch&&) = delete;
+    WriteBatch& operator=(WriteBatch&&) = delete;
+
+};
+
+}
+
+#endif

src/CMakeLists.txt

@@ -4,7 +4,8 @@ set(hepnos-src DataStore.cpp
 	       RunSet.cpp
 	       Run.cpp
 	       SubRun.cpp
-	       Event.cpp) 
+	       Event.cpp
+	       WriteBatch.cpp) 
 
 set(hepnos-service-src service/HEPnOSService.cpp
 		       service/ServiceConfig.cpp

src/DataSet.cpp

@@ -10,6 +10,7 @@
 #include "private/RunImpl.hpp"
 #include "private/DataSetImpl.hpp"
 #include "private/DataStoreImpl.hpp"
+#include "private/WriteBatchImpl.hpp"
 
 namespace hepnos {
 
@@ -93,7 +94,7 @@ bool DataSet::valid() const {
     return m_impl && m_impl->m_datastore; 
 }
 
-ProductID DataSet::storeRawData(const std::string& key, const std::vector<char>& buffer) {
+ProductID DataSet::storeRawData(const std::string& key, const std::string& buffer) {
     if(!valid()) {
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
@@ -101,7 +102,27 @@ ProductID DataSet::storeRawData(const std::string& key, const std::vector<char>&
     return m_impl->m_datastore->m_impl->store(0, fullname(), key, buffer);
 }
 
-bool DataSet::loadRawData(const std::string& key, std::vector<char>& buffer) const {
+ProductID DataSet::storeRawData(std::string&& key, std::string&& buffer) {
+    return storeRawData(key, buffer); // will call the function above
+}
+
+ProductID DataSet::storeRawData(WriteBatch& batch, const std::string& key, const std::string& buffer) {
+    if(!valid()) {
+        throw Exception("Calling DataSet member function on an invalid DataSet");
+    }
+    // forward the call to the datastore's store function
+    return batch.m_impl->store(0, fullname(), key, buffer);
+}
+
+ProductID DataSet::storeRawData(WriteBatch& batch, std::string&& key, std::string&& buffer) {
+    if(!valid()) {
+        throw Exception("Calling DataSet member function on an invalid DataSet");
+    }
+    // forward the call to the datastore's store function
+    return batch.m_impl->store(0, fullname(), std::move(key), std::move(buffer));
+}
+
+bool DataSet::loadRawData(const std::string& key, std::string& buffer) const {
     if(!valid()) {
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
@@ -153,7 +174,7 @@ DataSet DataSet::createDataSet(const std::string& name) {
         throw Exception("Invalid character '/' or '%' in dataset name");
     }
     std::string parent = fullname();
-    m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, name, std::vector<char>());
+    m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, name, std::string());
     return DataSet(m_impl->m_datastore, m_impl->m_level+1, parent, name);
 }
 
@@ -163,7 +184,7 @@ Run DataSet::createRun(const RunNumber& runNumber) {
     }
     std::string parent = fullname();
     std::string runStr = Run::Impl::makeKeyStringFromRunNumber(runNumber);
-    m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, runStr, std::vector<char>());
+    m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, runStr, std::string());
     return Run(m_impl->m_datastore, m_impl->m_level+1, parent, runNumber);
 }
 
@@ -206,9 +227,7 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) {
         datasetName   = datasetPath.substr(c+1);
     }
 
-    std::vector<char> data;
-
-    bool b = m_impl->m_datastore->m_impl->load(level, containerName, datasetName, data);
+    bool b = m_impl->m_datastore->m_impl->exists(level, containerName, datasetName);
     if(!b) {
         return m_impl->m_datastore->end();
     }

src/DataStore.cpp

@@ -9,7 +9,9 @@
 #include "hepnos/Exception.hpp"
 #include "hepnos/DataStore.hpp"
 #include "hepnos/DataSet.hpp"
+#include "hepnos/WriteBatch.hpp"
 #include "private/DataStoreImpl.hpp"
+#include "private/WriteBatchImpl.hpp"
 
 namespace hepnos {
 
@@ -74,7 +76,7 @@ DataStore::iterator DataStore::find(const std::string& datasetPath) {
         datasetName   = datasetPath.substr(c+1);
     }
 
-    std::vector<char> data;
+    std::string data;
     bool b = m_impl->load(level, containerName, datasetName, data);
     if(!b) {
         return m_impl->m_end;
@@ -175,7 +177,19 @@ DataSet DataStore::createDataSet(const std::string& name) {
     || name.find('%') != std::string::npos) {
         throw Exception("Invalid character ('/' or '%') in dataset name");
     }
-    m_impl->store(1, "", name, std::vector<char>());
+    m_impl->store(1, "", name, std::string());
+    return DataSet(this, 1, "", name);
+}
+
+DataSet DataStore::createDataSet(WriteBatch& batch, const std::string& name) {
+    if(!m_impl) {
+        throw Exception("Calling DataStore member function on an invalid DataStore object");
+    }
+    if(name.find('/') != std::string::npos
+    || name.find('%') != std::string::npos) {
+        throw Exception("Invalid character ('/' or '%') in dataset name");
+    }
+    batch.m_impl->store(1, "", name, std::string());
     return DataSet(this, 1, "", name);
 }
 
@@ -188,7 +202,7 @@ void DataStore::shutdown() {
     }
 }
 
-bool DataStore::loadRawProduct(const ProductID& productID, std::vector<char>& buffer) {
+bool DataStore::loadRawProduct(const ProductID& productID, std::string& buffer) {
     if(!m_impl) {
         throw Exception("Calling DataStore member function on an invalid DataStore object");
     }

src/Event.cpp

@@ -6,6 +6,7 @@
 #include "hepnos/Event.hpp"
 #include "private/EventImpl.hpp"
 #include "private/DataStoreImpl.hpp"
+#include "private/WriteBatchImpl.hpp"
 
 namespace hepnos {
 
@@ -64,7 +65,7 @@ bool Event::valid() const {
 
 }
 
-ProductID Event::storeRawData(const std::string& key, const std::vector<char>& buffer) {
+ProductID Event::storeRawData(const std::string& key, const std::string& buffer) {
     if(!valid()) {
         throw Exception("Calling Event member function on an invalid Event object");
     }
@@ -72,7 +73,27 @@ ProductID Event::storeRawData(const std::string& key, const std::vector<char>& b
     return m_impl->m_datastore->m_impl->store(0, m_impl->fullpath(), key, buffer);
 }
 
-bool Event::loadRawData(const std::string& key, std::vector<char>& buffer) const {
+ProductID Event::storeRawData(std::string&& key, std::string&& buffer) {
+    return storeRawData(key, buffer); // call the above function
+}
+
+ProductID Event::storeRawData(WriteBatch& batch, const std::string& key, const std::string& buffer) {
+    if(!valid()) {
+        throw Exception("Calling Event member function on an invalid Event object");
+    }
+    // forward the call to the datastore's store function
+    return batch.m_impl->store(0, m_impl->fullpath(), key, buffer);
+}
+
+ProductID Event::storeRawData(WriteBatch& batch, std::string&& key, std::string&& buffer) {
+    if(!valid()) {
+        throw Exception("Calling Event member function on an invalid Event object");
+    }
+    // forward the call to the datastore's store function
+    return batch.m_impl->store(0, m_impl->fullpath(), std::move(key), std::move(buffer));
+}
+
+bool Event::loadRawData(const std::string& key, std::string& buffer) const {
     if(!valid()) {
         throw Exception("Calling Event member function on an invalid Event object");
     }

src/Run.cpp

@@ -7,6 +7,7 @@
 #include "private/RunImpl.hpp"
 #include "private/SubRunImpl.hpp"
 #include "private/DataStoreImpl.hpp"
+#include "private/WriteBatchImpl.hpp"
 
 namespace hepnos {
 
@@ -66,7 +67,7 @@ bool Run::valid() const {
 
 }
 
-ProductID Run::storeRawData(const std::string& key, const std::vector<char>& buffer) {
+ProductID Run::storeRawData(const std::string& key, const std::string& buffer) {
     if(!valid()) {
         throw Exception("Calling Run member function on an invalid Run object");
     }
@@ -74,7 +75,27 @@ ProductID Run::storeRawData(const std::string& key, const std::vector<char>& buf
     return m_impl->m_datastore->m_impl->store(0, m_impl->fullpath(), key, buffer);
 }
 
-bool Run::loadRawData(const std::string& key, std::vector<char>& buffer) const {
+ProductID Run::storeRawData(std::string&& key, std::string&& buffer) {
+    return storeRawData(key, buffer); // calls the above function
+}
+
+ProductID Run::storeRawData(WriteBatch& batch, const std::string& key, const std::string& buffer) {
+    if(!valid()) {
+        throw Exception("Calling Run member function on an invalid Run object");
+    }
+    // forward the call to the datastore's store function
+    return batch.m_impl->store(0, m_impl->fullpath(), key, buffer);
+}
+
+ProductID Run::storeRawData(WriteBatch& batch, std::string&& key, std::string&& buffer) {
+    if(!valid()) {
+        throw Exception("Calling Run member function on an invalid Run object");
+    }
+    // forward the call to the datastore's store function
+    return batch.m_impl->store(0, m_impl->fullpath(), std::move(key), std::move(buffer));
+}
+
+bool Run::loadRawData(const std::string& key, std::string& buffer) const {
     if(!valid()) {
         throw Exception("Calling Run member function on an invalid Run object");
     }
@@ -118,7 +139,7 @@ SubRun Run::createSubRun(const SubRunNumber& subRunNumber) {
     }
     std::string parent = m_impl->fullpath();
     std::string subRunStr = SubRun::Impl::makeKeyStringFromSubRunNumber(subRunNumber);
-    m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, subRunStr, std::vector<char>());
+    m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, subRunStr, std::string());
     return SubRun(m_impl->m_datastore, m_impl->m_level+1, parent, subRunNumber);
 }
 
@@ -134,10 +155,9 @@ Run::iterator Run::find(const SubRunNumber& subRunNumber) {
         throw Exception("Calling Run member function on an invalid Run object");
     }
     int ret;
-    std::vector<char> data;
     std::string parent = m_impl->fullpath();
     std::string subRunStr = SubRun::Impl::makeKeyStringFromSubRunNumber(subRunNumber);
-    bool b = m_impl->m_datastore->m_impl->load(m_impl->m_level+1, parent, subRunStr, data);
+    bool b = m_impl->m_datastore->m_impl->exists(m_impl->m_level+1, parent, subRunStr);
     if(!b) {
         return m_impl->m_end;
     }

src/RunSet.cpp

@@ -34,12 +34,11 @@ Run RunSet::operator[](const RunNumber& runNumber) {
 
 RunSet::iterator RunSet::find(const RunNumber& runNumber) {
     int ret;
-    std::vector<char> data;
     std::string parent = m_impl->m_dataset->fullname();
     std::string strNum = Run::Impl::makeKeyStringFromRunNumber(runNumber);
     auto datastore = m_impl->m_dataset->m_impl->m_datastore;
     auto level = m_impl->m_dataset->m_impl->m_level;
-    bool b = datastore->m_impl->load(level+1, parent, strNum, data);
+    bool b = datastore->m_impl->exists(level+1, parent, strNum);
     if(!b) return end();
     return iterator(Run(datastore, level+1, parent, runNumber));
 }

src/SubRun.cpp

@@ -8,6 +8,7 @@
 #include "private/SubRunImpl.hpp"
 #include "private/EventImpl.hpp"
 #include "private/DataStoreImpl.hpp"
+#include "private/WriteBatchImpl.hpp"
 
 namespace hepnos {
 
@@ -67,7 +68,7 @@ bool SubRun::valid() const {
     return m_impl && m_impl->m_datastore; 
 }
 
-ProductID SubRun::storeRawData(const std::string& key, const std::vector<char>& buffer) {
+ProductID SubRun::storeRawData(const std::string& key, const std::string& buffer) {