improved memory utilization by using shared ptrs to strings

CMakeLists.txt

@@ -36,6 +36,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
 # packages we depend on
 include (xpkg-import)
 find_package (mercury CONFIG REQUIRED)
+find_package (thallium CONFIG REQUIRED)
 find_package (Boost REQUIRED COMPONENTS serialization)
 include_directories(${Boost_INCLUDE_DIRS})
 xpkg_import_module (margo REQUIRED margo)

bin/CMakeLists.txt

 add_executable(hepnos-daemon hepnos-daemon.cpp)
-target_link_libraries(hepnos-daemon hepnos-service yaml-cpp margo sdskv-server)
+target_link_libraries(hepnos-daemon hepnos-service yaml-cpp margo sdskv-server thallium)
 
 add_executable(hepnos-shutdown hepnos-shutdown.cpp)
 target_link_libraries(hepnos-shutdown hepnos yaml-cpp margo)

include/hepnos/DataSet.hpp

@@ -7,6 +7,7 @@
 #define __HEPNOS_DATA_SET_H
 
 #include <memory>
+#include <mpi.h>
 #include <hepnos/Exception.hpp>
 #include <hepnos/RunNumber.hpp>
 #include <hepnos/DataStore.hpp>
@@ -47,7 +48,7 @@ class DataSet : public KeyValueContainer {
      * @param container Full name of the parent DataSet ("" if no parent).
      * @param name Name of the DataSet.
      */
-    DataSet(DataStore* ds, uint8_t level, const std::string& container, const std::string& name);
+    DataSet(DataStore* ds, uint8_t level, const std::shared_ptr<std::string>& container, const std::string& name);
 
 
     /**
@@ -393,6 +394,19 @@ class DataSet : public KeyValueContainer {
      * @return a Run corresponding to the provided run number.
      */
     Run operator[](const RunNumber& runNumber) const;
+
+    /**
+     * @brief Executes a callback for each event in the hierarchy down
+     * from this DataSet. The events are dispatched to members of the comm
+     * MPI communicator.
+     *
+     * The callback should not make MPI calls.
+     *
+     * @param comm MPI communicator.
+     * @param callback Callback that will be called on each item.
+     */
+    void foreach(MPI_Comm comm, 
+            const std::function<void(const Run&, const SubRun&, const Event&)>& callback);
 };
 
 }

include/hepnos/Event.hpp

@@ -32,7 +32,7 @@ class Event : public KeyValueContainer {
      * @param container Full name of the container containing the event.
      * @param n Event number.
      */
-    Event(DataStore* datastore, uint8_t level, const std::string& container, const EventNumber& n);
+    Event(DataStore* datastore, uint8_t level, const std::shared_ptr<std::string>& container, const EventNumber& n);
 
     public:
 

include/hepnos/Run.hpp

@@ -36,7 +36,8 @@ class Run : public KeyValueContainer {
      * @param container Full name of the dataset containing the run.
      * @param run Run number.
      */
-    Run(DataStore* datastore, uint8_t level, const std::string& container, const RunNumber& run);
+    Run(DataStore* datastore, uint8_t level,
+            const std::shared_ptr<std::string>& container, const RunNumber& run);
 
     public:
 

include/hepnos/SubRun.hpp

@@ -33,7 +33,7 @@ class SubRun : public KeyValueContainer {
      * @param container Full name of the dataset containing the run.
      * @param run SubRun number.
      */
-    SubRun(DataStore* datastore, uint8_t level, const std::string& container, const SubRunNumber& run);
+    SubRun(DataStore* datastore, uint8_t level, const std::shared_ptr<std::string>& container, const SubRunNumber& run);
 
     public:
 

src/CMakeLists.txt

@@ -41,7 +41,7 @@ set_target_properties (hepnos
     SOVERSION ${HEPNOS_VERSION_MAJOR})
 
 add_library(hepnos-service ${hepnos-service-src})
-target_link_libraries (hepnos mercury margo yaml-cpp sdskv-client sdskv-server ch-placement)
+target_link_libraries (hepnos thallium mercury margo yaml-cpp sdskv-client sdskv-server ch-placement)
 target_include_directories (hepnos-service PUBLIC $<INSTALL_INTERFACE:include>)
 
 # local include's BEFORE, in case old incompatable .h files in prefix/include

src/DataSet.cpp

@@ -15,20 +15,20 @@
 namespace hepnos {
 
 DataSet::DataSet()
-: m_impl(std::make_unique<DataSet::Impl>(this, nullptr, 0, "", "")) {}
+: m_impl(std::make_unique<DataSet::Impl>(this, nullptr, 0, std::make_shared<std::string>(""), "")) {}
 
 DataSet::DataSet(DataStore* ds, uint8_t level, const std::string& fullname)
-: m_impl(std::make_unique<DataSet::Impl>(this, ds, level, "", "")) {
+: m_impl(std::make_unique<DataSet::Impl>(this, ds, level, std::make_shared<std::string>(""), "")) {
     size_t p = fullname.find_last_of('/');
     if(p == std::string::npos) {
         m_impl->m_name = fullname;
     } else {
         m_impl->m_name = fullname.substr(p+1);
-        m_impl->m_container = fullname.substr(0, p);
+        m_impl->m_container = std::make_shared<std::string>(fullname.substr(0, p));
     }
 }
 
-DataSet::DataSet(DataStore* ds, uint8_t level, const std::string& container, const std::string& name) 
+DataSet::DataSet(DataStore* ds, uint8_t level, const std::shared_ptr<std::string>& container, const std::string& name) 
 : m_impl(std::make_unique<DataSet::Impl>(this, ds, level, container, name)) {}
 
 DataSet::DataSet(const DataSet& other) {
@@ -85,7 +85,7 @@ DataSet DataSet::next() const {
    
     std::vector<std::string> keys; 
     size_t s = m_impl->m_datastore->m_impl->nextKeys(
-            m_impl->m_level, m_impl->m_container, m_impl->m_name, keys, 1);
+            m_impl->m_level, *m_impl->m_container, m_impl->m_name, keys, 1);
     if(s == 0) return DataSet();
     return DataSet(m_impl->m_datastore, m_impl->m_level, keys[0]);
 }
@@ -136,10 +136,10 @@ bool DataSet::operator==(const DataSet& other) const {
     if(!v1 && !v2) return true;
     if(v1  && !v2) return false;
     if(!v2 &&  v2) return false;
-    return m_impl->m_datastore == other.m_impl->m_datastore
-        && m_impl->m_level     == other.m_impl->m_level
-        && m_impl->m_container == other.m_impl->m_container
-        && m_impl->m_name      == other.m_impl->m_name;
+    return m_impl->m_datastore  == other.m_impl->m_datastore
+        && m_impl->m_level      == other.m_impl->m_level
+        && *m_impl->m_container == *other.m_impl->m_container
+        && m_impl->m_name       == other.m_impl->m_name;
 }
 
 bool DataSet::operator!=(const DataSet& other) const {
@@ -157,7 +157,7 @@ const std::string& DataSet::container() const {
     if(!valid()) {
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
-    return m_impl->m_container;
+    return *m_impl->m_container;
 }
 
 std::string DataSet::fullname() const {
@@ -175,7 +175,7 @@ DataSet DataSet::createDataSet(const std::string& name) {
     }
     std::string parent = fullname();
     m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, name, std::string());
-    return DataSet(m_impl->m_datastore, m_impl->m_level+1, parent, name);
+    return DataSet(m_impl->m_datastore, m_impl->m_level+1, std::make_shared<std::string>(parent), name);
 }
 
 Run DataSet::createRun(const RunNumber& runNumber) {
@@ -185,7 +185,8 @@ Run DataSet::createRun(const RunNumber& runNumber) {
     std::string parent = fullname();
     std::string runStr = Run::Impl::makeKeyStringFromRunNumber(runNumber);
     m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, runStr, std::string());
-    return Run(m_impl->m_datastore, m_impl->m_level+1, parent, runNumber);
+    return Run(m_impl->m_datastore, m_impl->m_level+1,
+            std::make_shared<std::string>(parent), runNumber);
 }
 
 Run DataSet::createRun(WriteBatch& batch, const RunNumber& runNumber) {
@@ -195,7 +196,8 @@ Run DataSet::createRun(WriteBatch& batch, const RunNumber& runNumber) {
     std::string parent = fullname();
     std::string runStr = Run::Impl::makeKeyStringFromRunNumber(runNumber);
     batch.m_impl->store(m_impl->m_level+1, parent, runStr, std::string());
-    return Run(m_impl->m_datastore, m_impl->m_level+1, parent, runNumber);
+    return Run(m_impl->m_datastore, m_impl->m_level+1,
+            std::make_shared<std::string>(parent), runNumber);
 }
 
 DataSet DataSet::operator[](const std::string& datasetName) const {
@@ -241,7 +243,7 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) {
     if(!b) {
         return m_impl->m_datastore->end();
     }
-    return iterator(DataSet(m_impl->m_datastore, level, containerName, datasetName));
+    return iterator(DataSet(m_impl->m_datastore, level, std::make_shared<std::string>(containerName), datasetName));
 }
 
 DataSet::const_iterator DataSet::find(const std::string& datasetName) const {
@@ -255,7 +257,7 @@ DataSet::iterator DataSet::begin() {
     }
     // we use the prefix "&" because we need something that comes after "%"
     // (which represents runs) and is not going to be in a dataset name
-    DataSet ds(m_impl->m_datastore, m_impl->m_level+1, fullname(),"&");
+    DataSet ds(m_impl->m_datastore, m_impl->m_level+1, std::make_shared<std::string>(fullname()),"&");
     ds = ds.next();
     if(ds.valid()) return iterator(ds);
     else return end();
@@ -303,7 +305,7 @@ DataSet::iterator DataSet::lower_bound(const std::string& lb) {
         ++it;
         return it;
     }
-    DataSet ds(m_impl->m_datastore, m_impl->m_level+1, fullname(), lb2);
+    DataSet ds(m_impl->m_datastore, m_impl->m_level+1, std::make_shared<std::string>(fullname()), lb2);
     ds = ds.next();
     if(!ds.valid()) return end();
     else return iterator(ds);
@@ -318,7 +320,7 @@ DataSet::iterator DataSet::upper_bound(const std::string& ub) {
     if(!valid()) {
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
-    DataSet ds(m_impl->m_datastore, m_impl->m_level+1, fullname(), ub);
+    DataSet ds(m_impl->m_datastore, m_impl->m_level+1, std::make_shared<std::string>(fullname()), ub);
     ds = ds.next();
     if(!ds.valid()) return end();
     else return iterator(ds);
@@ -340,4 +342,10 @@ const RunSet& DataSet::runs() const {
     return const_cast<DataSet*>(this)->runs();
 }
 
+void DataSet::foreach(MPI_Comm comm,
+        const std::function<void(const Run&, const SubRun&, const Event&)>& callback)
+{
+    // TODO
+}
+
 }

src/DataStore.cpp

@@ -81,7 +81,7 @@ DataStore::iterator DataStore::find(const std::string& datasetPath) {
     if(!b) {
         return m_impl->m_end;
     }
-    return iterator(DataSet(this, level, containerName, datasetName));
+    return iterator(DataSet(this, level, std::make_shared<std::string>(containerName), datasetName));
 }
 
 DataSet DataStore::operator[](const std::string& datasetName) const {
@@ -100,7 +100,7 @@ DataStore::iterator DataStore::begin() {
     if(!m_impl) {
         throw Exception("Calling DataStore member function on an invalid DataStore object");
     }
-    DataSet ds(this, 1, "", "");
+    DataSet ds(this, 1, std::make_shared<std::string>(""), "");
     ds = ds.next();
     if(ds.valid()) return iterator(std::move(ds));
     else return end();
@@ -143,7 +143,7 @@ DataStore::iterator DataStore::lower_bound(const std::string& lb) {
         ++it;
         return it;
     }
-    DataSet ds(this, 1, "", lb2);
+    DataSet ds(this, 1, std::make_shared<std::string>(""), lb2);
     ds = ds.next();
     if(!ds.valid()) return end();
     else return iterator(std::move(ds));
@@ -158,7 +158,7 @@ DataStore::iterator DataStore::upper_bound(const std::string& ub) {
     if(!m_impl) {
         throw Exception("Calling DataStore member function on an invalid DataStore object");
     }
-    DataSet ds(this, 1, "", ub);
+    DataSet ds(this, 1, std::make_shared<std::string>(""), ub);
     ds = ds.next();
     if(!ds.valid()) return end();
     else return iterator(std::move(ds));
@@ -178,7 +178,7 @@ DataSet DataStore::createDataSet(const std::string& name) {
         throw Exception("Invalid character ('/' or '%') in dataset name");
     }
     m_impl->store(1, "", name, std::string());
-    return DataSet(this, 1, "", name);
+    return DataSet(this, 1, std::make_shared<std::string>(""), name);
 }
 
 DataSet DataStore::createDataSet(WriteBatch& batch, const std::string& name) {
@@ -190,7 +190,7 @@ DataSet DataStore::createDataSet(WriteBatch& batch, const std::string& name) {
         throw Exception("Invalid character ('/' or '%') in dataset name");
     }
     batch.m_impl->store(1, "", name, std::string());
-    return DataSet(this, 1, "", name);
+    return DataSet(this, 1, std::make_shared<std::string>(""), name);
 }
 
 void DataStore::shutdown() {

src/Event.cpp

@@ -11,9 +11,9 @@
 namespace hepnos {
 
 Event::Event()
-: m_impl(std::make_unique<Impl>(nullptr, 0, "", InvalidEventNumber)) {} 
+: m_impl(std::make_unique<Impl>(nullptr, 0, std::make_shared<std::string>(""), InvalidEventNumber)) {} 
 
-Event::Event(DataStore* ds, uint8_t level, const std::string& container, const EventNumber& rn)
+Event::Event(DataStore* ds, uint8_t level, const std::shared_ptr<std::string>& container, const EventNumber& rn)
 : m_impl(std::make_unique<Impl>(ds, level, container, rn)) { }
 
 Event::Event(const Event& other) {
@@ -46,10 +46,10 @@ Event Event::next() const {
    
     std::vector<std::string> keys;
     size_t s = m_impl->m_datastore->m_impl->nextKeys(
-            m_impl->m_level, m_impl->m_container, 
+            m_impl->m_level, *m_impl->m_container, 
             m_impl->makeKeyStringFromEventNumber(), keys, 1);
     if(s == 0) return Event();
-    size_t i = m_impl->m_container.size()+1;
+    size_t i = m_impl->m_container->size()+1;
     if(keys[0].size() <= i) return Event();
     if(keys[0][i] != '%') return Event();
     std::stringstream strEventNumber;
@@ -107,10 +107,10 @@ bool Event::operator==(const Event& other) const {
     if(!v1 && !v2) return true;
     if(!v1 &&  v2) return false;
     if(v1  && !v2) return false;
-    return m_impl->m_datastore == other.m_impl->m_datastore
-        && m_impl->m_level     == other.m_impl->m_level
-        && m_impl->m_container == other.m_impl->m_container
-        && m_impl->m_event_nr  == other.m_impl->m_event_nr;
+    return m_impl->m_datastore  == other.m_impl->m_datastore
+        && m_impl->m_level      == other.m_impl->m_level
+        && *m_impl->m_container == *other.m_impl->m_container
+        && m_impl->m_event_nr   == other.m_impl->m_event_nr;
 }
 
 bool Event::operator!=(const Event& other) const {

src/Run.cpp

@@ -12,9 +12,9 @@
 namespace hepnos {
 
 Run::Run()
-: m_impl(std::make_unique<Run::Impl>(nullptr, 0, "", InvalidRunNumber)) {} 
+: m_impl(std::make_unique<Run::Impl>(nullptr, 0, std::make_shared<std::string>(""), InvalidRunNumber)) {} 
 
-Run::Run(DataStore* ds, uint8_t level, const std::string& container, const RunNumber& rn)
+Run::Run(DataStore* ds, uint8_t level, const std::shared_ptr<std::string>& container, const RunNumber& rn)
 : m_impl(std::make_unique<Run::Impl>(ds, level, container, rn)) { }
 
 Run::Run(const Run& other) {
@@ -48,10 +48,10 @@ Run Run::next() const {
    
     std::vector<std::string> keys;
     size_t s = m_impl->m_datastore->m_impl->nextKeys(
-            m_impl->m_level, m_impl->m_container, 
+            m_impl->m_level, *m_impl->m_container, 
             m_impl->makeKeyStringFromRunNumber(), keys, 1);
     if(s == 0) return Run();
-    size_t i = m_impl->m_container.size()+1;
+    size_t i = m_impl->m_container->size()+1;
     if(keys[0].size() <= i) return Run();
     if(keys[0][i] != '%') return Run();
     std::stringstream strRunNumber;
@@ -111,7 +111,7 @@ bool Run::operator==(const Run& other) const {
     if(!v1 && v2)  return false;
     return m_impl->m_datastore == other.m_impl->m_datastore
         && m_impl->m_level     == other.m_impl->m_level
-        && m_impl->m_container == other.m_impl->m_container
+        && *m_impl->m_container == *other.m_impl->m_container
         && m_impl->m_run_nr    == other.m_impl->m_run_nr;
 }
 
@@ -130,7 +130,7 @@ const std::string& Run::container() const {
     if(!valid()) {
         throw Exception("Calling Run member function on an invalid Run object");
     }
-    return m_impl->m_container;
+    return *m_impl->m_container;
 }
 
 SubRun Run::createSubRun(const SubRunNumber& subRunNumber) {
@@ -140,7 +140,8 @@ SubRun Run::createSubRun(const SubRunNumber& subRunNumber) {
     std::string parent = m_impl->fullpath();
     std::string subRunStr = SubRun::Impl::makeKeyStringFromSubRunNumber(subRunNumber);
     m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, subRunStr, std::string());
-    return SubRun(m_impl->m_datastore, m_impl->m_level+1, parent, subRunNumber);
+    return SubRun(m_impl->m_datastore, m_impl->m_level+1, 
+            std::make_shared<std::string>(parent), subRunNumber);
 }
 
 SubRun Run::createSubRun(WriteBatch& batch, const SubRunNumber& subRunNumber) {
@@ -150,7 +151,8 @@ SubRun Run::createSubRun(WriteBatch& batch, const SubRunNumber& subRunNumber) {
     std::string parent = m_impl->fullpath();
     std::string subRunStr = SubRun::Impl::makeKeyStringFromSubRunNumber(subRunNumber);
     batch.m_impl->store(m_impl->m_level+1, parent, subRunStr, std::string());
-    return SubRun(m_impl->m_datastore, m_impl->m_level+1, parent, subRunNumber);
+    return SubRun(m_impl->m_datastore, m_impl->m_level+1,
+            std::make_shared<std::string>(parent), subRunNumber);
 }
 
 SubRun Run::operator[](const SubRunNumber& subRunNumber) const {
@@ -171,7 +173,8 @@ Run::iterator Run::find(const SubRunNumber& subRunNumber) {
     if(!b) {
         return m_impl->m_end;
     }
-    return iterator(SubRun(m_impl->m_datastore, m_impl->m_level+1, parent, subRunNumber));
+    return iterator(SubRun(m_impl->m_datastore, m_impl->m_level+1,
+                std::make_shared<std::string>(parent), subRunNumber));
 }
 
 Run::const_iterator Run::find(const SubRunNumber& subRunNumber) const {
@@ -186,7 +189,7 @@ Run::iterator Run::begin() {
     auto level = m_impl->m_level;
     auto datastore = m_impl->m_datastore;
     std::string container = m_impl->fullpath();
-    SubRun subrun(datastore, level+1, container, 0);
+    SubRun subrun(datastore, level+1, std::make_shared<std::string>(container), 0);
     subrun = subrun.next();
 
     if(subrun.valid()) return iterator(subrun);
@@ -230,7 +233,7 @@ Run::iterator Run::lower_bound(const SubRunNumber& lb) {
         } else {
             SubRun subrun(m_impl->m_datastore, 
                     m_impl->m_level+1,
-                    m_impl->fullpath(), 0);
+                    std::make_shared<std::string>(m_impl->fullpath()), 0);
             subrun = subrun.next();
             if(!subrun.valid()) return end();
             else return iterator(subrun);
@@ -243,7 +246,7 @@ Run::iterator Run::lower_bound(const SubRunNumber& lb) {
         }
         SubRun subrun(m_impl->m_datastore,
                 m_impl->m_level+1,
-                m_impl->fullpath(), lb-1);
+                std::make_shared<std::string>(m_impl->fullpath()), lb-1);
         subrun = subrun.next();
         if(!subrun.valid()) return end();
         else return iterator(subrun);
@@ -261,7 +264,7 @@ Run::iterator Run::upper_bound(const SubRunNumber& ub) {
     }
     SubRun subrun(m_impl->m_datastore, 
             m_impl->m_level+1, 
-            m_impl->fullpath(), ub);
+            std::make_shared<std::string>(m_impl->fullpath()), ub);
     subrun = subrun.next();
     if(!subrun.valid()) return end();
     else return iterator(subrun);

src/RunSet.cpp

@@ -40,7 +40,7 @@ RunSet::iterator RunSet::find(const RunNumber& runNumber) {
     auto level = m_impl->m_dataset->m_impl->m_level;
     bool b = datastore->m_impl->exists(level+1, parent, strNum);
     if(!b) return end();
-    return iterator(Run(datastore, level+1, parent, runNumber));
+    return iterator(Run(datastore, level+1, std::make_shared<std::string>(parent), runNumber));
 }
 
 RunSet::const_iterator RunSet::find(const RunNumber& runNumber) const {
@@ -55,7 +55,7 @@ RunSet::iterator RunSet::begin() {
     auto ds_level = m_impl->m_dataset->m_impl->m_level;
     auto datastore = m_impl->m_dataset->m_impl->m_datastore;
     std::string container = m_impl->m_dataset->fullname();
-    Run run(datastore, ds_level+1, container, 0);
+    Run run(datastore, ds_level+1, std::make_shared<std::string>(container), 0);
     run = run.next();
 
     if(run.valid()) return iterator(run);
@@ -90,7 +90,7 @@ RunSet::iterator RunSet::lower_bound(const RunNumber& lb) {
         } else {
             Run run(m_impl->m_dataset->m_impl->m_datastore, 
                     m_impl->m_dataset->m_impl->m_level+1,
-                    m_impl->m_dataset->fullname(), 0);
+                    std::make_shared<std::string>(m_impl->m_dataset->fullname()), 0);
             run = run.next();
             if(!run.valid()) return end();
             else return iterator(run);
@@ -103,7 +103,7 @@ RunSet::iterator RunSet::lower_bound(const RunNumber& lb) {
         }
         Run run(m_impl->m_dataset->m_impl->m_datastore,
                 m_impl->m_dataset->m_impl->m_level+1,
-                m_impl->m_dataset->fullname(), lb-1);
+                std::make_shared<std::string>(m_impl->m_dataset->fullname()), lb-1);
         run = run.next();
         if(!run.valid()) return end();
         else return iterator(run);
@@ -118,7 +118,7 @@ RunSet::const_iterator RunSet::lower_bound(const RunNumber& lb) const {
 RunSet::iterator RunSet::upper_bound(const RunNumber& ub) {
     Run run(m_impl->m_dataset->m_impl->m_datastore, 
             m_impl->m_dataset->m_impl->m_level+1, 
-            m_impl->m_dataset->fullname(), ub);
+            std::make_shared<std::string>(m_impl->m_dataset->fullname()), ub);
     run = run.next();
     if(!run.valid()) return end();
     else return iterator(run);