big refactoring

include/hepnos/DataStore.hpp

@@ -21,9 +21,7 @@ class Event;
 class DataStoreImpl;
 class DataSetImpl;
 class RunSetImpl;
-class RunImpl;
-class SubRunImpl;
-class EventImpl;
+class ItemImpl;
 template<typename T, typename C = std::vector<T>> class Ptr;
 class WriteBatch;
 

include/hepnos/Event.hpp

@@ -21,13 +21,13 @@ class Event : public KeyValueContainer {
 
     friend class SubRun;
 
-    std::shared_ptr<EventImpl> m_impl;
+    std::shared_ptr<ItemImpl> m_impl;
 
     /**
      * @brief Constructor.
      */
-    Event(std::shared_ptr<EventImpl>&& impl);
-    Event(const std::shared_ptr<EventImpl>& impl);
+    Event(std::shared_ptr<ItemImpl>&& impl);
+    Event(const std::shared_ptr<ItemImpl>& impl);
 
     public:
 

include/hepnos/ProductID.hpp

@@ -14,27 +14,13 @@
 
 namespace hepnos {
 
-class WriteBatch;
-class DataStore;
-
 class ProductID {
 
     friend class DataStore;
     friend class DataStoreImpl;
-    friend class WriteBatch;
+    friend class WriteBatchImpl;
     friend class boost::serialization::access;
 
-    private:
-
-    std::uint8_t m_level;
-    std::string  m_containerName;
-    std::string  m_objectName;
-
-    ProductID(std::uint8_t level, const std::string& containerName, const std::string& objectName)
-    : m_level(level)
-    , m_containerName(containerName)
-    , m_objectName(objectName) {}
-
     public:
 
     ProductID() = default;
@@ -84,7 +70,7 @@ class ProductID {
      * underlying service, false otherwise.
      */
     bool valid() const {
-        return m_objectName.size() != 0;
+        return m_key.size() != 0;
     }
 
     /**
@@ -105,9 +91,7 @@ class ProductID {
      * @return true if the ProductIDs are the same, false otherwise.
      */
     bool operator==(const ProductID& other) const {
-        return m_level == other.m_level
-            && m_containerName == other.m_containerName
-            && m_objectName == other.m_objectName;
+        return m_key == other.m_key;
     }
 
     /**
@@ -123,11 +107,14 @@ class ProductID {
 
     private:
 
+    std::string m_key;
+
+    ProductID(const std::string& key)
+    : m_key(key) {}
+
     template<typename Archive>
     void serialize(Archive& ar, const unsigned int version) {
-        ar & m_level;
-        ar & m_containerName;
-        ar & m_objectName;
+        ar & m_key;
     }
 
 };

include/hepnos/Run.hpp

@@ -25,13 +25,13 @@ class Run : public KeyValueContainer {
     friend class RunSet;
     friend class DataSet;
 
-    std::shared_ptr<RunImpl> m_impl;
+    std::shared_ptr<ItemImpl> m_impl;
 
     /**
      * @brief Constructor.
      */
-    Run(const std::shared_ptr<RunImpl>& impl);
-    Run(std::shared_ptr<RunImpl>&& impl);
+    Run(const std::shared_ptr<ItemImpl>& impl);
+    Run(std::shared_ptr<ItemImpl>&& impl);
 
     public:
 

include/hepnos/SubRun.hpp

@@ -22,13 +22,13 @@ class SubRun : public KeyValueContainer {
 
     friend class Run;
 
-    std::shared_ptr<SubRunImpl> m_impl;
+    std::shared_ptr<ItemImpl> m_impl;
 
     /**
      * @brief Constructor.
      */
-    SubRun(std::shared_ptr<SubRunImpl>&& impl);
-    SubRun(const std::shared_ptr<SubRunImpl>& impl);
+    SubRun(std::shared_ptr<ItemImpl>&& impl);
+    SubRun(const std::shared_ptr<ItemImpl>& impl);
 
     public:
 

include/hepnos/UUID.hpp

@@ -46,6 +46,8 @@ struct UUID {
     void randomize();
 
     bool operator==(const UUID& other) const;
+
+    uint64_t hash() const;
 };
 
 template<typename T>

include/hepnos/WriteBatch.hpp

@@ -22,6 +22,7 @@ class DataSet;
 class Run;
 class SubRun;
 class Event;
+class WriteBatchImpl;
 
 class WriteBatch {
 
@@ -34,9 +35,7 @@ class WriteBatch {
 
     private:
 
-    class Impl;
-
-    std::unique_ptr<Impl> m_impl;
+    std::unique_ptr<WriteBatchImpl> m_impl;
 
     public:
 

src/CMakeLists.txt

@@ -10,6 +10,7 @@ set(hepnos-src DataStore.cpp
 
 set(hepnos-service-src service/HEPnOSService.cpp
 		       service/ServiceConfig.cpp
+		       UUID.cpp
 		       service/ConnectionInfoGenerator.cpp)
 
 # load package helper for generating cmake CONFIG packages
@@ -28,7 +29,7 @@ set (hepnos-vers "${HEPNOS_VERSION_MAJOR}.${HEPNOS_VERSION_MINOR}")
 set (HEPNOS_VERSION "${hepnos-vers}.${HEPNOS_VERSION_PATCH}")
 
 add_library(hepnos ${hepnos-src})
-target_link_libraries (hepnos uuid mercury margo yaml-cpp sdskv-client ch-placement)
+target_link_libraries (hepnos uuid thallium mercury margo yaml-cpp sdskv-client ch-placement)
 target_include_directories (hepnos PUBLIC $<INSTALL_INTERFACE:include>)
 
 # local include's BEFORE, in case old incompatable .h files in prefix/include
@@ -42,7 +43,7 @@ set_target_properties (hepnos
     SOVERSION ${HEPNOS_VERSION_MAJOR})
 
 add_library(hepnos-service ${hepnos-service-src})
-target_link_libraries (hepnos uuid thallium mercury margo yaml-cpp sdskv-client sdskv-server ch-placement)
+target_link_libraries (hepnos-service uuid thallium mercury margo yaml-cpp sdskv-client sdskv-server ch-placement)
 target_include_directories (hepnos-service PUBLIC $<INSTALL_INTERFACE:include>)
 
 # local include's BEFORE, in case old incompatable .h files in prefix/include

src/DataSet.cpp

@@ -6,7 +6,7 @@
 #include "hepnos/DataSet.hpp"
 #include "hepnos/Run.hpp"
 #include "hepnos/RunSet.hpp"
-#include "RunImpl.hpp"
+#include "ItemImpl.hpp"
 #include "DataSetImpl.hpp"
 #include "DataStoreImpl.hpp"
 #include "WriteBatchImpl.hpp"
@@ -48,7 +48,8 @@ ProductID DataSet::storeRawData(const std::string& key, const char* value, size_
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
     // forward the call to the datastore's store function
-    return m_impl->m_datastore->store(0, fullname(), key, value, vsize);
+    ItemDescriptor id(m_impl->m_uuid);
+    return m_impl->m_datastore->storeRawProduct(id, key, value, vsize);
 }
 
 ProductID DataSet::storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) {
@@ -56,7 +57,8 @@ ProductID DataSet::storeRawData(WriteBatch& batch, const std::string& key, const
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
     // forward the call to the datastore's store function
-    return batch.m_impl->store(0, fullname(), key, value, vsize);
+    ItemDescriptor id(m_impl->m_uuid);
+    return batch.m_impl->storeRawProduct(id, key, value, vsize);
 }
 
 bool DataSet::loadRawData(const std::string& key, std::string& buffer) const {
@@ -64,7 +66,8 @@ bool DataSet::loadRawData(const std::string& key, std::string& buffer) const {
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
     // forward the call to the datastore's load function
-    return m_impl->m_datastore->load(0, fullname(), key, buffer);
+    ItemDescriptor id(m_impl->m_uuid);
+    return m_impl->m_datastore->loadRawProduct(id, key, buffer);
 }
 
 bool DataSet::loadRawData(const std::string& key, char* value, size_t* vsize) const {
@@ -72,7 +75,8 @@ bool DataSet::loadRawData(const std::string& key, char* value, size_t* vsize) co
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
     // forward the call to the datastore's load function
-    return m_impl->m_datastore->load(0, fullname(), key, value, vsize);
+    ItemDescriptor id(m_impl->m_uuid);
+    return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
 }
 
 bool DataSet::operator==(const DataSet& other) const {
@@ -130,12 +134,9 @@ Run DataSet::createRun(const RunNumber& runNumber) {
     if(InvalidRunNumber == runNumber) {
         throw Exception("Trying to create a Run with InvalidRunNumber");
     }
-    std::string parent_uuid = m_impl->m_uuid.to_string();
-    std::string runStr = makeKeyStringFromNumber(runNumber);
-    m_impl->m_datastore->store(m_impl->m_level+1, parent_uuid, runStr);
-    return Run(std::make_shared<RunImpl>(
+    m_impl->m_datastore->createItem(m_impl->m_uuid, runNumber);
+    return Run(std::make_shared<ItemImpl>(
                     m_impl->m_datastore,
-                    m_impl->m_level+1,
                     m_impl->m_uuid,
                     runNumber));
 }
@@ -144,13 +145,11 @@ Run DataSet::createRun(WriteBatch& batch, const RunNumber& runNumber) {
     if(InvalidRunNumber == runNumber) {
         throw Exception("Trying to create a Run with InvalidRunNumber");
     }
-    std::string parent_uuid = m_impl->m_uuid.to_string();
-    std::string runStr = makeKeyStringFromNumber(runNumber);
-    batch.m_impl->store(m_impl->m_level+1, parent_uuid, runStr);
-    return Run(
-            std::make_shared<RunImpl>(
-                m_impl->m_datastore, m_impl->m_level+1,
-                m_impl->m_uuid, runNumber));
+    batch.m_impl->createItem(m_impl->m_uuid, runNumber);
+    return Run(std::make_shared<ItemImpl>(
+                    m_impl->m_datastore,
+                    m_impl->m_uuid,
+                    runNumber));
 }
 
 DataSet DataSet::operator[](const std::string& datasetName) const {

src/DataStore.cpp

@@ -63,14 +63,14 @@ bool DataStore::loadRawProduct(const ProductID& productID, std::string& buffer)
     if(!m_impl) {
         throw Exception("Calling DataStore member function on an invalid DataStore object");
     }
-    return m_impl->load(productID.m_level, productID.m_containerName, productID.m_objectName, buffer);
+    return m_impl->loadRawProduct(productID, buffer);
 }
 
 bool DataStore::loadRawProduct(const ProductID& productID, char* data, size_t* size) {
     if(!m_impl) {
         throw Exception("Calling DataStore member function on an invalid DataStore object");
     }
-    return m_impl->load(productID.m_level, productID.m_containerName, productID.m_objectName, data, size);
+    return m_impl->loadRawProduct(productID, data, size);
 }
 
 }

src/Event.cpp

@@ -4,19 +4,19 @@
  * See COPYRIGHT in top-level directory.
  */
 #include "hepnos/Event.hpp"
-#include "EventImpl.hpp"
+#include "ItemImpl.hpp"
 #include "DataStoreImpl.hpp"
 #include "WriteBatchImpl.hpp"
 
 namespace hepnos {
 
 Event::Event()
-: m_impl(std::make_shared<EventImpl>(0, nullptr, InvalidEventNumber)) {} 
+: m_impl(std::make_shared<ItemImpl>(nullptr, UUID(), InvalidRunNumber)) {} 
 
-Event::Event(std::shared_ptr<EventImpl>&& impl)
+Event::Event(std::shared_ptr<ItemImpl>&& impl)
 : m_impl(std::move(impl)) { }
 
-Event::Event(const std::shared_ptr<EventImpl>& impl)
+Event::Event(const std::shared_ptr<ItemImpl>& impl)
 : m_impl(impl) { }
 
 DataStore Event::datastore() const {
@@ -28,18 +28,11 @@ DataStore Event::datastore() const {
 
 Event Event::next() const {
     if(!valid()) return Event();
-   
-    std::vector<std::string> keys;
-    auto container = m_impl->container();
-    size_t s = m_impl->m_datastore->nextKeys(
-            m_impl->m_level, container, 
-            m_impl->makeKeyStringFromEventNumber(), keys, 1);
+
+    std::vector<std::shared_ptr<ItemImpl>> next_events;
+    size_t s = m_impl->m_datastore->nextItems(m_impl, next_events, 1);
     if(s == 0) return Event();
-    size_t i = container.size()+1;
-    if(keys[0].size() <= i) return Event();
-    EventNumber n = parseNumberFromKeyString<EventNumber>(&keys[0][i]);
-    if(n == InvalidEventNumber) return Event();
-    return Event(std::make_shared<EventImpl>(m_impl->m_level, m_impl->m_subrun, n));
+    return Event(std::move(next_events[0]));   
 }
 
 bool Event::valid() const {
@@ -52,7 +45,8 @@ ProductID Event::storeRawData(const std::string& key, const char* value, size_t
         throw Exception("Calling Event member function on an invalid Event object");
     }
     // forward the call to the datastore's store function
-    return m_impl->m_datastore->store(0, m_impl->fullpath(), key, value, vsize);
+    auto& id = m_impl->m_descriptor;
+    return m_impl->m_datastore->storeRawProduct(id, key, value, vsize);
 }
 
 ProductID Event::storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) {
@@ -60,7 +54,8 @@ ProductID Event::storeRawData(WriteBatch& batch, const std::string& key, const c
         throw Exception("Calling Event member function on an invalid Event object");
     }
     // forward the call to the datastore's store function
-    return batch.m_impl->store(0, m_impl->fullpath(), key, value, vsize);
+    auto& id = m_impl->m_descriptor;
+    return batch.m_impl->storeRawProduct(id, key, value, vsize);
 }
 
 bool Event::loadRawData(const std::string& key, std::string& buffer) const {
@@ -68,7 +63,8 @@ bool Event::loadRawData(const std::string& key, std::string& buffer) const {
         throw Exception("Calling Event member function on an invalid Event object");
     }
     // forward the call to the datastore's load function
-    return m_impl->m_datastore->load(0, m_impl->fullpath(), key, buffer);
+    auto& id = m_impl->m_descriptor;
+    return m_impl->m_datastore->loadRawProduct(id, key, buffer);
 }
 
 bool Event::loadRawData(const std::string& key, char* value, size_t* vsize) const {
@@ -76,7 +72,8 @@ bool Event::loadRawData(const std::string& key, char* value, size_t* vsize) cons
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
     // forward the call to the datastore's load function
-    return m_impl->m_datastore->load(0, m_impl->fullpath(), key, value, vsize);
+    auto& id = m_impl->m_descriptor;
+    return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
 }
 
 bool Event::operator==(const Event& other) const {
@@ -96,7 +93,7 @@ const EventNumber& Event::number() const {
     if(!valid()) {
         throw Exception("Calling Event member function on an invalid Event object");
     }
-    return m_impl->m_event_number;
+    return m_impl->m_descriptor.event;
 }
 
 }

src/EventImpl.hpp

-/*
- * (C) 2018 The University of Chicago
- * 
- * See COPYRIGHT in top-level directory.
- */
-#ifndef __HEPNOS_PRIVATE_EVENT_IMPL_H
-#define __HEPNOS_PRIVATE_EVENT_IMPL_H
-
-#include <boost/predef/other/endian.h>
-#include <sstream>
-#include <iomanip>
-#include <memory>
-#include "DataStoreImpl.hpp"
-#include "hepnos/Run.hpp"
-#include "hepnos/SubRun.hpp"
-#include "hepnos/Event.hpp"
-#include "NumberUtil.hpp"
-#include "SubRunImpl.hpp"
-
-namespace hepnos {
-
-class EventImpl {
-
-    public:
-
-        std::shared_ptr<DataStoreImpl> m_datastore;
-        std::shared_ptr<SubRunImpl>    m_subrun;
-        EventNumber                    m_event_number;
-        uint8_t                        m_level;
-
-        EventImpl(uint8_t level,
-                  const std::shared_ptr<SubRunImpl>& subrun,
-                  const EventNumber& evn)
-        : m_subrun(subrun)
-        , m_event_number(evn)
-        , m_level(level) {
-            if(subrun) m_datastore = subrun->m_datastore;
-        }
-
-        bool operator==(const EventImpl& other) const {
-            if(m_event_number != other.m_event_number) return false;
-            if(m_subrun == other.m_subrun) return true;
-            return *m_subrun == *other.m_subrun;
-        }
-
-        std::string makeKeyStringFromEventNumber() const {
-            return makeKeyStringFromNumber(m_event_number);
-        }
-
-        std::string container() const {
-            return m_subrun->fullpath();
-        }
-
-        std::string fullpath() const {
-            return container() + "/" + makeKeyStringFromEventNumber();
-        }
-};
-
-}
-
-#endif

src/ItemDescriptor.hpp

+#ifndef __HEPNOS_ITEM_DESCRIPTOR_HPP
+#define __HEPNOS_ITEM_DESCRIPTOR_HPP
+
+#include "hepnos/RunNumber.hpp"
+#include "hepnos/SubRunNumber.hpp"
+#include "hepnos/EventNumber.hpp"
+#include "hepnos/UUID.hpp"
+
+namespace hepnos {
+
+struct ItemDescriptor {
+    UUID         dataset;
+    RunNumber    run;
+    SubRunNumber subrun;
+    EventNumber  event;
+
+    ItemDescriptor()
+    : run(InvalidRunNumber)
+    , subrun(InvalidSubRunNumber)
+    , event(InvalidEventNumber) {
+        std::memset(dataset.data, '\0', sizeof(dataset.data));
+    }
+
+    ItemDescriptor(const UUID& ds,
+                   const RunNumber& rn = InvalidRunNumber,
+                   const SubRunNumber& srn = InvalidSubRunNumber,
+                   const EventNumber& evn = InvalidEventNumber)
+    : dataset(ds)
+    , run(rn)
+    , subrun(srn)
+    , event(evn) {}
+
+    bool operator==(const ItemDescriptor& other) const {
+        return dataset == other.dataset
+            && run     == other.run
+            && subrun  == other.subrun
+            && event   == other.event;
+    }
+
+    bool operator!=(const ItemDescriptor& other) const {
+        return !(*this == other);
+    }
+
+    bool operator<(const ItemDescriptor& other) const {
+        // check the level of each ItemDescriptor
+        uint8_t level = 2;
+        if(event == InvalidEventNumber) {
+            if(subrun == InvalidSubRunNumber) level = 0;
+            else level = 1;
+        }
+        uint8_t other_level = 2;
+        if(other.event == InvalidEventNumber) {
+            if(other.subrun == InvalidSubRunNumber) other_level = 0;
+            else other_level = 1;
+        }
+        if(level < other_level) return true;
+        if(level > other_level) return false;
+        // here we know the descriptors have the same level, so we compare datasets
+        int c = memcmp(dataset.data, other.dataset.data, sizeof(dataset));
+        if(c < 0) return true;
+        if(c > 0) return false;
+        // datasets are the same, we compare run numbers
+        if(run < other.run) return true;
+        if(run > other.run) return false;
+        // runs are the same, we compare subrun numbers
+        if(subrun == InvalidSubRunNumber) return false;
+        if(subrun < other.subrun) return true;
+        if(subrun > other.subrun) return false;
+        // subruns are the same, we compare event numbers
+        if(event < other.event) return true;
+        return false;
+    }
+
+    template<typename S>
+    friend S& operator<<(S& s, const ItemDescriptor& d);
+};
+
+template<typename S>
+S& operator<<(S& s, const ItemDescriptor& d) {
+    s << "[" << d.dataset.to_string() << ", "
+      << d.run << ", "
+      << d.subrun << ", "
+      << d.event << "]";
+    return s;
+}
+
+}
+
+#endif

src/ItemImpl.hpp

+/*
+ * (C) 2018 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
+#ifndef __HEPNOS_PRIVATE_ITEM_IMPL_H
+#define __HEPNOS_PRIVATE_ITEM_IMPL_H
+
+#include <sstream>
+#include <iomanip>
+#include <memory>
+#include "hepnos/Run.hpp"
+#include "hepnos/UUID.hpp"
+#include "NumberUtil.hpp"
+#include "ItemDescriptor.hpp"
+
+namespace hepnos {
+
+class DataStoreImpl;
+
+class ItemImpl {
+
+    public:
+
+        std::shared_ptr<DataStoreImpl> m_datastore;
+        ItemDescriptor                 m_descriptor;
+
+        ItemImpl(const std::shared_ptr<DataStoreImpl>& ds,
+                 const UUID& dataset,
+                 const RunNumber& rn,
+                 const SubRunNumber& srn = InvalidSubRunNumber,
+                 const EventNumber& evn = InvalidEventNumber)
+        : m_datastore(ds)
+        , m_descriptor(dataset, rn, srn, evn) {}
+
+        ItemImpl(const std::shared_ptr<DataStoreImpl>& ds,
+                 const ItemDescriptor& descriptor)
+        : m_datastore(ds)
+        , m_descriptor(descriptor) {}
+
+        bool operator==(const ItemImpl& other) const {
+            return m_descriptor == other.m_descriptor;
+        }
+
+        size_t parentPrefixSize() const {
+            size_t s = sizeof(ItemDescriptor) - sizeof(m_descriptor.event);
+            if(m_descriptor.event == InvalidEventNumber) 
+                s -= sizeof(m_descriptor.subrun);
+            if(m_descriptor.subrun == InvalidSubRunNumber)
+                s -= sizeof(m_descriptor.run);
+            return s;
+        }
+};
+
+}
+
+#endif

src/Run.cpp

@@ -4,22 +4,22 @@
  * See COPYRIGHT in top-level directory.
  */
 #include "hepnos/Run.hpp"
-#include "RunImpl.hpp"
-#include "SubRunImpl.hpp"
+#include "ItemImpl.hpp"
+#include "ItemImpl.hpp"
 #include "DataStoreImpl.hpp"
 #include "WriteBatchImpl.hpp"
 
 namespace hepnos {
 
-Run::iterator RunImpl::m_end;
+static Run::iterator Run_end;
 
 Run::Run()
-: m_impl(std::make_shared<RunImpl>(nullptr, 0, UUID(), InvalidRunNumber)) {} 
+: m_impl(std::make_shared<ItemImpl>(nullptr, UUID(), InvalidRunNumber)) {} 
 
-Run::Run(std::shared_ptr<RunImpl>&& impl)
+Run::Run(std::shared_ptr<ItemImpl>&& impl)
 : m_impl(std::move(impl)) { }
 
-Run::Run(const std::shared_ptr<RunImpl>& impl)
+Run::Run(const std::shared_ptr<ItemImpl>& impl)
 : m_impl(impl) { }
 
 DataStore Run::datastore() const {
@@ -32,20 +32,10 @@ DataStore Run::datastore() const {
 Run Run::next() const {
     if(!valid()) return Run();
    
-    std::vector<std::string> keys;
-    size_t s = m_impl->m_datastore->nextKeys(
-            m_impl->m_level, m_impl->m_dataset_uuid.to_string(), 
-            m_impl->makeKeyStringFromRunNumber(), keys, 1);
+    std::vector<std::shared_ptr<ItemImpl>> next_runs;
+    size_t s = m_impl->m_datastore->nextItems(m_impl, next_runs, 1);
     if(s == 0) return Run();
-    size_t i = 33;
-    if(keys[0].size() <= i) return Run();
-    RunNumber rn = parseNumberFromKeyString<RunNumber>(&keys[0][i]);
-    if(rn == InvalidRunNumber) return Run();
-    return Run(std::make_shared<RunImpl>(
-               m_impl->m_datastore,
-               m_impl->m_level,
-               m_impl->m_dataset_uuid, 
-               rn));
+    return Run(std::move(next_runs[0]));
 }
 
 bool Run::valid() const {
@@ -58,7 +48,8 @@ ProductID Run::storeRawData(const std::string& key, const char* value, size_t vs
         throw Exception("Calling Run member function on an invalid Run object");
     }
     // forward the call to the datastore's store function
-    return m_impl->m_datastore->store(0, m_impl->fullpath(), key, value, vsize);
+    const ItemDescriptor& id = m_impl->m_descriptor;
+    return m_impl->m_datastore->storeRawProduct(id, key, value, vsize);
 }
 
 ProductID Run::storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) {
@@ -66,7 +57,8 @@ ProductID Run::storeRawData(WriteBatch& batch, const std::string& key, const cha
         throw Exception("Calling Run member function on an invalid Run object");
     }
     // forward the call to the datastore's store function
-    return batch.m_impl->store(0, m_impl->fullpath(), key, value, vsize);
+    const ItemDescriptor& id = m_impl->m_descriptor;
+    return batch.m_impl->storeRawProduct(id, key, value, vsize);
 }
 
 bool Run::loadRawData(const std::string& key, std::string& buffer) const {
@@ -74,7 +66,8 @@ bool Run::loadRawData(const std::string& key, std::string& buffer) const {
         throw Exception("Calling Run member function on an invalid Run object");
     }
     // forward the call to the datastore's load function
-    return m_impl->m_datastore->load(0, m_impl->fullpath(), key, buffer);
+    const ItemDescriptor& id = m_impl->m_descriptor;
+    return m_impl->m_datastore->loadRawProduct(id, key, buffer);
 }
 
 bool Run::loadRawData(const std::string& key, char* value, size_t* vsize) const {
@@ -82,7 +75,8 @@ bool Run::loadRawData(const std::string& key, char* value, size_t* vsize) const
         throw Exception("Calling DataSet member function on an invalid DataSet");
     }
     // forward the call to the datastore's load function
-    return m_impl->m_datastore->load(0, m_impl->fullpath(), key, value, vsize);
+    const ItemDescriptor& id = m_impl->m_descriptor;
+    return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
 }
 
 bool Run::operator==(const Run& other) const {
@@ -102,17 +96,16 @@ const RunNumber& Run::number() const {
     if(!valid()) {
         throw Exception("Calling Run member function on an invalid Run object");
     }
-    return m_impl->m_run_number;
+    return m_impl->m_descriptor.run;
 }
 
 SubRun Run::createSubRun(const SubRunNumber& subRunNumber) {
     if(!valid()) {
         throw Exception("Calling Run member function on an invalid Run object");
     }
-    std::string parent = m_impl->fullpath();
-    std::string subRunStr = makeKeyStringFromNumber(subRunNumber);
-    m_impl->m_datastore->store(m_impl->m_level+1, parent, subRunStr);
-    auto new_subrun_impl = std::make_shared<SubRunImpl>(m_impl->m_level+1, m_impl, subRunNumber);
+    ItemDescriptor& id = m_impl->m_descriptor;
+    m_impl->m_datastore->createItem(id.dataset, id.run, subRunNumber);
+    auto new_subrun_impl = std::make_shared<ItemImpl>(m_impl->m_datastore, id.dataset, id.run, subRunNumber);
     return SubRun(std::move(new_subrun_impl));
 }
 
@@ -120,10 +113,9 @@ SubRun Run::createSubRun(WriteBatch& batch, const SubRunNumber& subRunNumber) {
     if(!valid()) {
         throw Exception("Calling Run member function on an invalid Run object");
     }
-    std::string parent = m_impl->fullpath();
-    std::string subRunStr = makeKeyStringFromNumber(subRunNumber);
-    batch.m_impl->store(m_impl->m_level+1, parent, subRunStr);
-    auto new_subrun_impl = std::make_shared<SubRunImpl>(m_impl->m_level+1, m_impl, subRunNumber);
+    ItemDescriptor& id = m_impl->m_descriptor;
+    batch.m_impl->createItem(id.dataset, id.run, subRunNumber);
+    auto new_subrun_impl = std::make_shared<ItemImpl>(m_impl->m_datastore, id.dataset, id.run, subRunNumber);
     return SubRun(std::move(new_subrun_impl));
 }
 
@@ -138,14 +130,14 @@ Run::iterator Run::find(const SubRunNumber& subRunNumber) {
     if(!valid()) {
         throw Exception("Calling Run member function on an invalid Run object");