implemented a way to prefetch objects during iterations

include/hepnos/DataSet.hpp

@@ -164,6 +164,8 @@ class DataSet : public KeyValueContainer {
      */
     bool loadRawData(const std::string& key, std::string& value) const override;
     bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& value) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
 
     /**
      * @brief Comparison operator.

include/hepnos/Event.hpp

@@ -128,6 +128,8 @@ class Event : public KeyValueContainer {
      */
     bool loadRawData(const std::string& key, std::string& value) const override;
     bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
 
     /**
      * @brief Compares this Event with another Event. The Events must point to

include/hepnos/KeyValueContainer.hpp

@@ -22,6 +22,7 @@ namespace hepnos {
 
 class WriteBatch;
 class AsyncEngine;
+class Prefetcher;
 
 class KeyValueContainer {
 
@@ -104,6 +105,10 @@ class KeyValueContainer {
 
     virtual bool loadRawData(const std::string& key, char* value, size_t* vsize) const = 0;
 
+    virtual bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const = 0;
+
+    virtual bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const = 0;
+
     /**
      * @brief Stores a key/value pair into the KeyValueContainer.
      * The type of the key should have operator<< available
@@ -207,6 +212,16 @@ class KeyValueContainer {
         return loadVectorImpl(key, value, std::is_pod<V>());
     }
 
+    template<typename K, typename V>
+    bool load(const Prefetcher& prefetcher, const K& key, V& value) const {
+        return loadImpl(prefetcher, key, value, std::is_pod<V>());
+    }
+
+    template<typename K, typename V>
+    bool load(const Prefetcher& prefetcher, const K& key, std::vector<V>& value) const {
+        return loadVectorImpl(prefetcher, key, value, std::is_pod<V>());
+    }
+
     private:
 
     /**
@@ -297,6 +312,19 @@ class KeyValueContainer {
         return vsize == sizeof(value);
     }
 
+    template<typename K, typename V>
+    bool loadImpl(const Prefetcher& prefetcher, const K& key, V& value,
+            const std::integral_constant<bool, true>&) const {
+        std::string buffer;
+        std::stringstream ss_key;
+        ss_key << key << "#" << demangle<V>();
+        size_t vsize = sizeof(value);
+        if(!loadRawData(prefetcher, ss_key.str(), reinterpret_cast<char*>(&value), &vsize)) {
+            return false;
+        }
+        return vsize == sizeof(value);
+    }
+
     /**
      * @brief Implementation of the load function when the value type is not a POD.
      */
@@ -319,6 +347,25 @@ class KeyValueContainer {
         return true;
     }
 
+    template<typename K, typename V>
+    bool loadImpl(const Prefetcher& prefetcher, const K& key, V& value,
+            const std::integral_constant<bool, false>&) const {
+        std::string buffer;
+        std::stringstream ss_key;
+        ss_key << key << "#" << demangle<V>();
+        if(!loadRawData(prefetcher, ss_key.str(), buffer)) {
+            return false;
+        }
+        try {
+            std::stringstream ss(buffer);
+            InputArchive ia(datastore(), ss);
+            ia >> value;
+        } catch(...) {
+            throw Exception("Exception occured during serialization");
+        }
+        return true;
+    }
+
     /**
      * @brief Implementation of the load function when the value type is a vector of POD.
      */
@@ -344,6 +391,28 @@ class KeyValueContainer {
         return true;
     }
 
+    template<typename K, typename V>
+    bool loadVectorImpl(const Prefetcher& prefetcher, const K& key, std::vector<V>& value,
+            const std::integral_constant<bool, true>&) const {
+        std::string buffer;
+        std::stringstream ss_key;
+        ss_key << key << "#" << demangle<std::vector<V>>();
+        if(!loadRawData(prefetcher, ss_key.str(), buffer)) {
+            return false;
+        }
+        size_t count = 0;
+        if(buffer.size() < sizeof(count)) {
+            return false;
+        }
+        std::memcpy(&count, buffer.data(), sizeof(count));
+        if(buffer.size() != sizeof(count) + count*sizeof(V)) {
+            return false;
+        }
+        value.resize(count);
+        std::memcpy(value.data(), buffer.data()+sizeof(count), count*sizeof(V));
+        return true;
+    }
+
     /**
      * @brief Implementation of the load function when the value type is a vector of non-POD.
      */
@@ -370,6 +439,31 @@ class KeyValueContainer {
         }
         return true;
     }
+
+    template<typename K, typename V>
+    bool loadVectorImpl(const Prefetcher& prefetcher, const K& key, std::vector<V>& value,
+            const std::integral_constant<bool, false>&) const {
+        std::string buffer;
+        std::stringstream ss_key;
+        ss_key << key << "#" << demangle<std::vector<V>>();
+        if(!loadRawData(prefetcher, ss_key.str(), buffer)) {
+            return false;
+        }
+        try {
+            std::stringstream ss(buffer);
+            InputArchive ia(datastore(), ss);
+            size_t count = 0;
+            ia >> count;
+            value.resize(count);
+            for(unsigned i=0; i<count; i++) {
+                ia >> value[i];
+            }
+        } catch(...) {
+            throw Exception("Exception occured during serialization");
+        }
+        return true;
+    }
+
     /**
      * @brief Creates the string key based on the provided key
      * and the type of the value. Serializes the value into a string.

include/hepnos/Prefetcher.hpp

@@ -2,6 +2,7 @@
 #define __HEPNOS_PREFETCHER_HPP
 
 #include <memory>
+#include <hepnos/Demangle.hpp>
 #include <hepnos/Prefetchable.hpp>
 
 namespace hepnos {
@@ -13,6 +14,7 @@ class RunSet;
 class EventSet;
 class Run;
 class SubRun;
+class Event;
 
 class Prefetcher {
 
@@ -20,10 +22,7 @@ class Prefetcher {
     friend class EventSet;
     friend class Run;
     friend class SubRun;
-
-    private:
-
-    std::shared_ptr<PrefetcherImpl> m_impl;
+    friend class Event;
 
     public:
 
@@ -48,6 +47,18 @@ class Prefetcher {
     Prefetchable<Container> operator()(const Container& c) const {
         return Prefetchable<Container>(c, *this);
     }
+
+    template<typename V>
+    void fetchProduct(const std::string& label, bool fetch=true) const {
+        fetchProductImpl(label + "#" + demangle<V>(), fetch);
+    }
+
+    private:
+
+    std::shared_ptr<PrefetcherImpl> m_impl;
+
+    void fetchProductImpl(const std::string& labelAndType, bool fetch) const;
+
 };
 
 }

include/hepnos/ProductID.hpp

@@ -20,6 +20,7 @@ class ProductID {
     friend class DataStoreImpl;
     friend class AsyncEngineImpl;
     friend class WriteBatchImpl;
+    friend class PrefetcherImpl;
     friend class boost::serialization::access;
 
     public:

include/hepnos/Run.hpp

@@ -132,6 +132,8 @@ class Run : public KeyValueContainer {
      */
     bool loadRawData(const std::string& key, std::string& buffer) const override;
     bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
 
     /**
      * @brief Compares this Run with another Run. The Runs must point to

include/hepnos/SubRun.hpp

@@ -130,6 +130,8 @@ class SubRun : public KeyValueContainer {
      */
     bool loadRawData(const std::string& key, std::string& buffer) const override;
     bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const override;
+    bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
 
     /**
      * @brief Compares this SubRun with another SubRun. The SubRuns must point to

src/DataSet.cpp

@@ -7,12 +7,14 @@
 #include "hepnos/Run.hpp"
 #include "hepnos/RunSet.hpp"
 #include "hepnos/AsyncEngine.hpp"
+#include "hepnos/Prefetcher.hpp"
 #include "ItemImpl.hpp"
 #include "DataSetImpl.hpp"
 #include "EventSetImpl.hpp"
 #include "DataStoreImpl.hpp"
 #include "AsyncEngineImpl.hpp"
 #include "WriteBatchImpl.hpp"
+#include "PrefetcherImpl.hpp"
 
 namespace hepnos {
 
@@ -97,6 +99,24 @@ bool DataSet::loadRawData(const std::string& key, char* value, size_t* vsize) co
     return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
 }
 
+bool DataSet::loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const {
+    if(!valid()) {
+        throw Exception("Calling DataSet member function on an invalid DataSet");
+    }
+    (void)prefetcher; // prefetcher isn't usable with a DataSet
+    ItemDescriptor id(m_impl->m_uuid);
+    return m_impl->m_datastore->loadRawProduct(id, key, buffer);
+}
+
+bool DataSet::loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const {
+    if(!valid()) {
+        throw Exception("Calling DataSet member function on an invalid DataSet");
+    }
+    (void)prefetcher; // prefetcher isn't usable with a DataSet 
+    ItemDescriptor id(m_impl->m_uuid);
+    return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
+}
+
 bool DataSet::operator==(const DataSet& other) const {
     bool v1 = valid();
     bool v2 = other.valid();

src/Event.cpp

@@ -5,10 +5,12 @@
  */
 #include "hepnos/Event.hpp"
 #include "hepnos/AsyncEngine.hpp"
+#include "hepnos/Prefetcher.hpp"
 #include "ItemImpl.hpp"
 #include "DataStoreImpl.hpp"
 #include "WriteBatchImpl.hpp"
 #include "AsyncEngineImpl.hpp"
+#include "PrefetcherImpl.hpp"
 
 namespace hepnos {
 
@@ -104,6 +106,22 @@ bool Event::loadRawData(const std::string& key, char* value, size_t* vsize) cons
     return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
 }
 
+bool Event::loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const {
+    if(!valid()) {
+        throw Exception("Calling Event member function on an invalid Event object");
+    }
+    auto& id = m_impl->m_descriptor;
+    return prefetcher.m_impl->loadRawProduct(id, key, buffer);
+}
+
+bool Event::loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const {
+    if(!valid()) {
+        throw Exception("Calling DataSet member function on an invalid DataSet");
+    }
+    auto& id = m_impl->m_descriptor;
+    return prefetcher.m_impl->loadRawProduct(id, key, value, vsize);
+}
+
 bool Event::operator==(const Event& other) const {
     bool v1 = valid();
     bool v2 = other.valid();

src/Prefetcher.cpp

@@ -34,4 +34,12 @@ void Prefetcher::setBatchSize(unsigned int size) {
     m_impl->m_batch_size = size;
 }
 
+void Prefetcher::fetchProductImpl(const std::string& label, bool fetch=true) const {
+    if(fetch) {
+        m_impl->m_active_product_keys.insert(label);
+    } else {
+        m_impl->m_active_product_keys.erase(label);
+    }
+}
+
 }

src/PrefetcherImpl.hpp

@@ -2,6 +2,8 @@
 #define __HEPNOS_PREFETCHER_IMPL_HPP
 
 #include <set>
+#include <unordered_set>
+#include <unordered_map>
 #include "DataStoreImpl.hpp"
 #include "AsyncEngineImpl.hpp"
 
@@ -22,8 +24,10 @@ class PrefetcherImpl {
     std::shared_ptr<AsyncEngineImpl> m_async;
     unsigned int                     m_cache_size = 16;
     unsigned int                     m_batch_size = 1;
-    mutable std::set<std::shared_ptr<ItemImpl>, ItemPtrComparator> m_cache;
     bool                             m_associated = false;
+    std::unordered_set<std::string>  m_active_product_keys;
+    mutable std::set<std::shared_ptr<ItemImpl>, ItemPtrComparator> m_item_cache;
+    mutable std::unordered_map<std::string, std::string> m_product_cache;
 
     PrefetcherImpl(const std::shared_ptr<DataStoreImpl>& ds)
     : m_datastore(ds) {}
@@ -33,19 +37,35 @@ class PrefetcherImpl {
     : m_datastore(ds)
     , m_async(async) {}
 
+    void fetchRequestedProducts(const std::shared_ptr<ItemImpl>& itemImpl) const {
+        auto& descriptor = itemImpl->m_descriptor;
+        for(auto& key : m_active_product_keys) {
+            auto product_id = DataStoreImpl::buildProductID(descriptor, key);
+            auto it = m_product_cache.find(product_id.m_key);
+            if(it != m_product_cache.end())
+                continue;
+            std::string data;
+            bool ok = m_datastore->loadRawProduct(product_id, data);
+            if(ok) {
+                m_product_cache[product_id.m_key] = std::move(data);
+            }
+        }
+    }
+
     void prefetchFrom(const ItemType& item_type,
             const ItemType& prefix_type,
             const std::shared_ptr<ItemImpl>& current,
             int target=-1) const
     {
         auto last = current;
-        while(m_cache.size() != m_cache_size) {
+        while(m_item_cache.size() != m_cache_size) {
             std::vector<std::shared_ptr<ItemImpl>> items;
             size_t s = m_datastore->nextItems(item_type, prefix_type, last, items, m_batch_size, target);
             if(s != 0)
                 last = items[items.size()-1];
             for(auto& item : items) {
-                m_cache.insert(std::move(item));
+                fetchRequestedProducts(item);
+                m_item_cache.insert(std::move(item));
             }
             if(s < m_batch_size) break;
         }
@@ -59,25 +79,53 @@ class PrefetcherImpl {
             size_t maxItems,
             int target=-1) const
     {
-        auto ub = m_cache.upper_bound(current);
-        if(ub == m_cache.end()) {
-            m_cache.clear();
+        auto ub = m_item_cache.upper_bound(current);
+        if(ub == m_item_cache.end()) {
+            m_item_cache.clear();
             prefetchFrom(item_type, prefix_type, current, target);
         }
-        ub = m_cache.upper_bound(current);
+        ub = m_item_cache.upper_bound(current);
         result.clear();
-        if(ub == m_cache.end()) {
+        if(ub == m_item_cache.end()) {
             return 0;
         } else {
             auto it = ub;
             result.clear();
-            for(size_t i=0; i < maxItems && it != m_cache.end(); i++, it++) {
+            for(size_t i=0; i < maxItems && it != m_item_cache.end(); i++, it++) {
                 result.push_back(*it);
             }
-            m_cache.erase(ub, it);
+            m_item_cache.erase(ub, it);
         }
         return result.size();
     }
+
+    bool loadRawProduct(const ItemDescriptor& id,
+                        const std::string& productName,
+                        std::string& data) const {
+        auto product_id = DataStoreImpl::buildProductID(id, productName);
+        auto it = m_product_cache.find(product_id.m_key);
+        if(it == m_product_cache.end()) {
+            return m_datastore->loadRawProduct(product_id, data);
+        } else {
+            data = std::move(it->second);
+            m_product_cache.erase(it);
+            return true;
+        }
+    }
+
+    bool loadRawProduct(const ItemDescriptor& id,
+                        const std::string& productName,
+                        char* value, size_t* vsize) const {
+        auto product_id = DataStoreImpl::buildProductID(id, productName);
+        auto it = m_product_cache.find(product_id.m_key);
+        if(it == m_product_cache.end()) {
+            return m_datastore->loadRawProduct(id, productName, value, vsize);
+        } else {
+            *vsize = it->second.size();
+            std::memcpy(value, it->second.data(), *vsize);
+            return true;
+        }
+    }
 };
 
 }

src/Run.cpp

@@ -146,6 +146,22 @@ bool Run::loadRawData(const std::string& key, char* value, size_t* vsize) const
     return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
 }
 
+bool Run::loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const {
+    if(!valid()) {
+        throw Exception("Calling Run member function on an invalid Run object");
+    }
+    const ItemDescriptor& id = m_impl->m_descriptor;
+    return prefetcher.m_impl->loadRawProduct(id, key, buffer);
+}
+
+bool Run::loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const {
+    if(!valid()) {
+        throw Exception("Calling DataSet member function on an invalid DataSet");
+    }
+    const ItemDescriptor& id = m_impl->m_descriptor;
+    return prefetcher.m_impl->loadRawProduct(id, key, value, vsize);
+}
+
 bool Run::operator==(const Run& other) const {
     bool v1 = valid();
     bool v2 = other.valid();
@@ -228,6 +244,7 @@ Run::iterator Run::find(const SubRunNumber& subRunNumber, const Prefetcher& pref
     auto it = find(subRunNumber);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_subrun.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::SUBRUN, ItemType::RUN, it.m_impl->m_current_subrun.m_impl);
     }
     return it;
@@ -256,6 +273,7 @@ Run::iterator Run::begin(const Prefetcher& prefetcher) {
     auto it = begin();
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_subrun.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::SUBRUN, ItemType::RUN, it.m_impl->m_current_subrun.m_impl);
     }
     return it;
@@ -330,6 +348,7 @@ Run::iterator Run::lower_bound(const SubRunNumber& lb, const Prefetcher& prefetc
     auto it = lower_bound(lb);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_subrun.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::SUBRUN, ItemType::RUN, it.m_impl->m_current_subrun.m_impl);
     }
     return it;
@@ -359,6 +378,7 @@ Run::iterator Run::upper_bound(const SubRunNumber& ub, const Prefetcher& prefetc
     auto it = upper_bound(ub);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_subrun.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::SUBRUN, ItemType::RUN, it.m_impl->m_current_subrun.m_impl);
     }
     return it;

src/RunSet.cpp

@@ -136,6 +136,7 @@ RunSet::iterator RunSet::begin(const Prefetcher& prefetcher) {
     auto it = begin();
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_run.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::RUN, ItemType::DATASET, it.m_impl->m_current_run.m_impl);
     }
     return it;
@@ -153,6 +154,7 @@ RunSet::const_iterator RunSet::cbegin(const Prefetcher& prefetcher) const {
     auto it = cbegin();
     if(it != cend()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_run.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::RUN, ItemType::DATASET, it.m_impl->m_current_run.m_impl);
     }
     return it;
@@ -170,6 +172,7 @@ RunSet::const_iterator RunSet::begin(const Prefetcher& prefetcher) const {
     auto it = const_iterator(const_cast<RunSet*>(this)->begin());
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_run.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::RUN, ItemType::DATASET, it.m_impl->m_current_run.m_impl);
     }
     return it;
@@ -211,6 +214,7 @@ RunSet::iterator RunSet::lower_bound(const RunNumber& lb, const Prefetcher& pref
     auto it = lower_bound(lb);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_run.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::RUN, ItemType::DATASET, it.m_impl->m_current_run.m_impl);
     }
     return it;
@@ -225,6 +229,7 @@ RunSet::const_iterator RunSet::lower_bound(const RunNumber& lb, const Prefetcher
     iterator it = const_cast<RunSet*>(this)->lower_bound(lb);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_run.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::RUN, ItemType::DATASET, it.m_impl->m_current_run.m_impl);
     }
     return it;
@@ -242,6 +247,7 @@ RunSet::iterator RunSet::upper_bound(const RunNumber& ub, const Prefetcher& pref
     auto it = upper_bound(ub);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_run.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::RUN, ItemType::DATASET, it.m_impl->m_current_run.m_impl);
     }
     return it;
@@ -257,6 +263,7 @@ RunSet::const_iterator RunSet::upper_bound(const RunNumber& ub, const Prefetcher
     iterator it = const_cast<RunSet*>(this)->upper_bound(ub);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_run.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::RUN, ItemType::DATASET, it.m_impl->m_current_run.m_impl);
     }
     return it;

src/SubRun.cpp

@@ -158,6 +158,22 @@ bool SubRun::loadRawData(const std::string& key, char* value, size_t* vsize) con
     return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
 }
 
+bool SubRun::loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const {
+    if(!valid()) {
+        throw Exception("Calling SubRun member function on invalid SubRun object");
+    }
+    auto& id = m_impl->m_descriptor;
+    return prefetcher.m_impl->loadRawProduct(id, key, buffer);
+}
+
+bool SubRun::loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const {
+    if(!valid()) {
+        throw Exception("Calling DataSet member function on an invalid DataSet");
+    }
+    auto& id = m_impl->m_descriptor;
+    return prefetcher.m_impl->loadRawProduct(id, key, value, vsize);
+}
+
 bool SubRun::operator==(const SubRun& other) const {
     bool v1 = valid();
     bool v2 = other.valid();
@@ -231,6 +247,7 @@ SubRun::iterator SubRun::find(const EventNumber& eventNumber, const Prefetcher&
     auto it = find(eventNumber);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_event.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::EVENT, ItemType::SUBRUN, it.m_impl->m_current_event.m_impl);
     }
     return it;
@@ -261,6 +278,7 @@ SubRun::iterator SubRun::begin(const Prefetcher& prefetcher) {
     auto it = begin();
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_event.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::EVENT, ItemType::SUBRUN, it.m_impl->m_current_event.m_impl);
     }
     return it;
@@ -333,6 +351,7 @@ SubRun::iterator SubRun::lower_bound(const EventNumber& lb, const Prefetcher& pr
     auto it = lower_bound(lb);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_event.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::EVENT, ItemType::SUBRUN, it.m_impl->m_current_event.m_impl);
     }
     return it;
@@ -361,6 +380,7 @@ SubRun::iterator SubRun::upper_bound(const EventNumber& lb, const Prefetcher& pr
     auto it = upper_bound(lb);
     if(it != end()) {
         it.m_impl->setPrefetcher(prefetcher.m_impl);
+        prefetcher.m_impl->fetchRequestedProducts(it.m_impl->m_current_event.m_impl);
         prefetcher.m_impl->prefetchFrom(ItemType::EVENT, ItemType::SUBRUN, it.m_impl->m_current_event.m_impl);
     }
     return it;

test/LoadStoreTest.cpp

@@ -409,3 +409,41 @@ void LoadStoreTest::testAsyncLoadStoreEvent() {
     // and they are the same
     CPPUNIT_ASSERT(in_obj_b == out_obj_b);
 }
+
+void LoadStoreTest::testPrefetchLoadStore() {
+    auto root = datastore->root();
+    auto mds = root.createDataSet("prefetch_run");
+    std::string label = "key";
+    {
+        TestObjectA obj_a;
+        TestObjectB obj_b;
+        for(unsigned i = 0; i < 10; i++) {
+            obj_a.x() = i;
+            obj_a.y() = 2*i;
+            obj_b.a() = 3*i;
+            obj_b.b() = "matthieu";
+            auto r = mds.createRun(i);
+            CPPUNIT_ASSERT(r.valid());
+            r.store(label, obj_a);
+            r.store(label, obj_b);
+        }
+    }
+    {
+        // iterate through the dataset with a prefetcher
+        Prefetcher p(*datastore);
+        p.fetchProduct<TestObjectA>(label);
+        p.fetchProduct<TestObjectB>(label);
+        unsigned i = 0;
+        for(auto& run : p(mds.runs())) {
+            TestObjectA obj_a;
+            TestObjectB obj_b;
+            run.load(p, label, obj_a);
+            run.load(p, label, obj_b);
+            CPPUNIT_ASSERT(obj_a.x() == i);
+            CPPUNIT_ASSERT(obj_a.y() == 2*i);
+            CPPUNIT_ASSERT(obj_b.a() == 3*i);
+            CPPUNIT_ASSERT(obj_b.b() == "matthieu");
+            i += 1;
+        }
+    }
+}

test/LoadStoreTest.hpp

@@ -18,6 +18,7 @@ class LoadStoreTest : public CppUnit::TestFixture
     CPPUNIT_TEST( testAsyncLoadStoreRun );
     CPPUNIT_TEST( testAsyncLoadStoreSubRun );
     CPPUNIT_TEST( testAsyncLoadStoreEvent );
+    CPPUNIT_TEST( testPrefetchLoadStore );
     CPPUNIT_TEST_SUITE_END();
 
     public:
@@ -34,6 +35,7 @@ class LoadStoreTest : public CppUnit::TestFixture
     void testAsyncLoadStoreRun();
     void testAsyncLoadStoreSubRun();
     void testAsyncLoadStoreEvent();
+    void testPrefetchLoadStore();
 };
 
 #endif