better comments

include/hepnos.hpp

+/*
+ * (C) 2018 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
 #ifndef __HEPNOS_H
 #define __HEPNOS_H
 

include/hepnos/AsyncEngine.hpp

@@ -22,6 +22,16 @@ class Prefetcher;
 class PrefetcherImpl;
 class AsyncEngineImpl;
 
+/**
+ * @brief The AsyncEngine class uses Argobots to provide a set
+ * of execution streams (ES) to perform operations in the background.
+ * This object may then be used by Prefetcher or WriteBatch
+ * instances to perform their RPCs in the background.
+ * The AsyncEngine may be initialized with 0 ES, in which case
+ * the communications will still happen in the background using
+ * Mercury's non-blocking functions, but the rest of the processing
+ * will happen synchronously.
+ */
 class AsyncEngine {
 
     friend class DataStore;
@@ -39,14 +49,72 @@ class AsyncEngine {
 
     public:
 
+    /**
+     * @brief Constructor.
+     *
+     * @param ds DataStore instance.
+     * @param num_threads Number of execution streams (background threads) to start.
+     */
     AsyncEngine(DataStore& ds, size_t num_threads=0);
+
+    /**
+     * @brief Destructor.
+     */
     ~AsyncEngine() = default;
+
+    /**
+     * @brief Copy constructor. A copy of an existing
+     * AsyncEngine will share the same pool of threads as the
+     * instance it was copied from.
+     *
+     * @param AsyncEngine AsyncEngine instance to copy.
+     */
     AsyncEngine(const AsyncEngine&) = default;
+
+    /**
+     * @brief Copy-assignment operator. A copy of
+     * an existing AsyncEngine will share the same pool of
+     * threads as the instance it was copied from.
+     *
+     * @param AsyncEngine AsyncEngine to copy.
+     *
+     * @return The assigned-to instance.
+     */
     AsyncEngine& operator=(const AsyncEngine&) = default;
+
+    /**
+     * @brief Move constructor. The thread pool will be
+     * moved to the new AsyncEngine instance, leaving the
+     * moved-from instance in an invalid state.
+     *
+     * @param AsyncEngine AsyncEngine to move from.
+     */
     AsyncEngine(AsyncEngine&&) = default;
+
+    /**
+     * @brief Move-assignment operator. The thread pool
+     * will be moved to the new moved-to AsyncEngine
+     * instance, leaving the moved-from instance in an
+     * invalid state.
+     *
+     * @param AsyncEngine AsyncEngine to move from.
+     *
+     * @return The moved-to instance.
+     */
     AsyncEngine& operator=(AsyncEngine&&) = default;
 
+    /**
+     * @brief Blocks until the threads in this AsyncEngine
+     * don't have any more work to do.
+     */
     void wait();
+
+    /**
+     * @brief Returns the list of errors that occured
+     * when running the asynchronous work.
+     *
+     * @return List of error messages.
+     */
     const std::vector<std::string>& errors() const;
 };
 

include/hepnos/DataSet.hpp

@@ -32,9 +32,6 @@ class DataSet : public KeyValueContainer {
 
     private:
 
-    /**
-     * @brief Implementation class (used for the Pimpl idiom).
-     */
     std::shared_ptr<DataSetImpl> m_impl; /*!< Pointer to implementation. */
 
     /**
@@ -137,34 +134,38 @@ class DataSet : public KeyValueContainer {
     bool valid() const;
 
     /**
-     * @brief Stores binary data associated with a particular key into this DataSet.
-     * This function will return true if the key did not already exist and the
-     * write succeeded. It will return false otherwise.
-     *
-     * @param key Key.
-     * @param buffer Binary data to insert.
-     *
-     * @return a valid ProductID if the key did not already exist and the write succeeded,
-     *      an invalid one otherwise.
+     * @see KeyValueContainer::storeRawData()
      */
     ProductID storeRawData(const std::string& key, const char* value, size_t vsize) override;
+
+    /**
+     * @see KeyValueContainer::storeRawData()
+     */
     ProductID storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) override;
+
+    /**
+     * @see KeyValueContainer::storeRawData()
+     */
     ProductID storeRawData(AsyncEngine& engine, const std::string& key, const char* value, size_t vsize) override;
 
     /**
-     * @brief Loads binary data associated with a particular key from the DataSet.
-     * This function will return true if the key exists and the read succeeded.
-     * It will return false otherwise.
-     * 
-     * @param key Key.
-     * @param buffer Buffer in which to put the binary data.
-     *
-     * @return true if the key exists and the read succeeded,
-     *      false otherwise.
+     * @see KeyValueContainer::loadRawData()
      */
     bool loadRawData(const std::string& key, std::string& value) const override;
+
+    /**
+     * @see KeyValueContainer::loadRawData()
+     */
     bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
+
+    /**
+     * @see KeyValueContainer::loadRawData()
+     */
     bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& value) const override;
+
+    /**
+     * @see KeyValueContainer::loadRawData()
+     */
     bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
 
     /**
@@ -213,7 +214,31 @@ class DataSet : public KeyValueContainer {
      * @return A Run instance pointing to the created run.
      */
     Run createRun(const RunNumber& runNumber);
+
+    /**
+     * @brief Creates a Run asynchronously.
+     * Note that even though the returned Run object is valid, since
+     * the actual creation operation will happen in the background,
+     * there is no guarantee that this operation will succeed.
+     *
+     * @param async AsyncEngine to use to make the operation happen in the background.
+     * @param runNumber Run number.
+     *
+     * @return a valid Run object.
+     */
     Run createRun(AsyncEngine& async, const RunNumber& runNumber);
+
+    /**
+     * @brief Creates a Run by pushing the creation operation into
+     * a WriteBatch. Note that even though the returned Run object is valid,
+     * since the actual creation is delayed until the WriteBatch is flushed,
+     * there is no guarantee that this operation will ultimately succeed.
+     *
+     * @param batch WriteBatch in which to append the operation.
+     * @param runNumber Run number.
+     *
+     * @return a valid Run object.
+     */
     Run createRun(WriteBatch& batch, const RunNumber& runNumber);
 
     /**

include/hepnos/DataStore.hpp

@@ -182,9 +182,6 @@ class DataStore {
 
     private:
 
-    /**
-     * @brief Implementation of the class (using Pimpl idiom)
-     */
     std::shared_ptr<DataStoreImpl> m_impl; /*!< Pointer to implementation */
 
     /**

include/hepnos/Demangle.hpp

@@ -11,12 +11,23 @@
 
 namespace hepnos {
 
+/**
+ * @brief Returns the name of a type.
+ *
+ * @tparam T Type from which to return the name.
+ *
+ * @return Name of type T.
+ */
 template<typename T>
 std::string demangle() {
     char const * name = typeid(T).name();    
     return boost::core::demangle(name);
 }
 
+/**
+ * @brief Returns the name of type, which example
+ * instance provided for type deduction.
+ */
 template<typename T>
 std::string demangle(T&&) {
     return demangle<T>();

include/hepnos/Event.hpp

@@ -24,6 +24,10 @@ struct EventDescriptor {
     char data[EventDescriptorLength];
 };
 
+/**
+ * @brief The Event class is a KeyValueContainer contained in SubRuns
+ * and with no sub-container.
+ */
 class Event : public KeyValueContainer {
 
     private:
@@ -107,28 +111,38 @@ class Event : public KeyValueContainer {
     bool valid() const;
 
     /**
-     * @brief Stores raw key/value data in this Event.
-     *
-     * @param key Key
-     * @param value Value
-     *
-     * @return a valid ProductID if the key did not already exist, an invalid one otherwise.
+     * @see KeyValueContainer::storeRawData()
      */
     ProductID storeRawData(const std::string& key, const char* value, size_t vsize) override;
+
+    /**
+     * @see KeyValueContainer::storeRawData()
+     */
     ProductID storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) override;
+
+    /**
+     * @see KeyValueContainer::storeRawData()
+     */
     ProductID storeRawData(AsyncEngine& async, const std::string& key, const char* value, size_t vsize) override;
 
     /**
-     * @brief Loads raw key/value data from this Event.
-     *
-     * @param key Key
-     * @param value Buffer used to hold the value.
-     *
-     * @return true if the key exists, false otherwise.
+     * @see KeyValueContainer::loadRawData()
      */
     bool loadRawData(const std::string& key, std::string& value) const override;
+
+    /**
+     * @see KeyValueContainer::loadRawData()
+     */
     bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
+
+    /**
+     * @see KeyValueContainer::loadRawData()
+     */
     bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const override;
+
+    /**
+     * @see KeyValueContainer::loadRawData()
+     */
     bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
 
     /**

include/hepnos/Exception.hpp

@@ -11,17 +11,42 @@
 
 namespace hepnos {
 
+/**
+ * @brief Exception class thrown by HEPnOS functions on failure.
+ */
 class Exception : public std::exception
 {
     std::string m_msg;
 
     public:
 
+    /**
+     * @brief Default constructor.
+     */
     Exception() = default;
+
+    /**
+     * @brief Constructor.
+     *
+     * @param msg Error message.
+     */
     Exception(const std::string& msg) : m_msg(msg){}
+    
+    /**
+     * @brief Copy constructor.
+     */
     Exception(const Exception&) = default;
+
+    /**
+     * @brief Copy-assignment operator.
+     */
     Exception& operator=(const Exception&) = default;
 
+    /**
+     * @brief Returns the error message.
+     *
+     * @return the error message.
+     */
     virtual const char* what() const noexcept override
     {
         return m_msg.c_str();

include/hepnos/InputArchive.hpp

@@ -34,17 +34,34 @@ class InputArchive : public iarchive {
 
     public:
 
+    /**
+     * @brief Constructor.
+     *
+     * @tparam Args Argument types necessary for the underlying boost iarchive.
+     * @param datastore DataStore.
+     * @param args Arguments necessary for the parent boost iarchive.
+     */
     template<typename ... Args>
     InputArchive(DataStore&& datastore, Args&& ... args)
     : iarchive(std::forward<Args>(args)..., boost::archive::archive_flags::no_header)
     , m_datastore(std::move(datastore)) {}
 
+    /**
+     * @brief Constructor.
+     *
+     * @tparam Args Argument types necessary for the underlying boost iarchive.
+     * @param datastore DataStore.
+     * @param args Arguments necessary for the parent boost iarchive.
+     */
     template<typename ... Args>
     InputArchive(const DataStore& datastore, Args&& ... args)
     : iarchive(std::forward<Args>(args)..., boost::archive::archive_flags::no_header)
     , m_datastore(datastore) {}
 
-    DataStore datastore() const {
+    /**
+     * @brief Returns the DataStore used when creating the InputArchive.
+     */
+    const DataStore& datastore() const {
         return m_datastore;
     }
 

include/hepnos/ItemType.hpp

+/*
+ * (C) 2018 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
 #ifndef __HEPNOS_ITEM_TYPE_HPP
 #define __HEPNOS_ITEM_TYPE_HPP
 
 namespace hepnos {
 
+/**
+ * @brief Type of item (DataSet, Run, SubRun, Event, or Product).
+ */
 enum class ItemType : uint32_t {
     DATASET,
     RUN,

include/hepnos/KeyValueContainer.hpp

@@ -81,15 +81,34 @@ class KeyValueContainer {
      * @brief Stores raw key/value data in a WriteBatch.
      * This function is virtual and must be overloaded in the child class.
      *
+     * Note since the WriteBatch is flushed later to the DataStore, the DataStore will
+     * not be able to check whether the product could be created or not. Hence the
+     * ProductID returned is valid but may not ultimately correspond to an actual
+     * Product in the DataStore, should the storage operation fail.
+     *
      * @param batch Batch in which to write.
      * @param key Key
      * @param value Value pointer
      * @param vsize Value size (in bytes)
      *
-     * @return 
+     * @return A valid ProductID.
      */
     virtual ProductID storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) = 0;
 
+    /**
+     * @brief Stores binary data associated with a particular key using the AsyncEngine.
+     * Note since the AsyncEngine makes the operation happen later in the background,
+     * the DataStore will not be able to check whether the product could be created or not.
+     * Hence the ProductID returned is valid but may not ultimately correspond to an actual
+     * Product in the DataStore, should the storage operation fail.
+     *
+     * @param engine AsyncEngine to use to write asynchronously.
+     * @param key Key.
+     * @param value Binary data to write.
+     * @param vsize Size of the data (in bytes).
+     *
+     * @return a valid ProductID.
+     */
     virtual ProductID storeRawData(AsyncEngine& async, const std::string& key, const char* value, size_t vsize) = 0;
 
     /**
@@ -103,10 +122,45 @@ class KeyValueContainer {
      */
     virtual bool loadRawData(const std::string& key, std::string& buffer) const = 0;
 
+    /**
+     * @brief Loads binary data associated with a particular key from the container.
+     * This function will return true if the key exists and the read succeeded.
+     * It will return false otherwise.
+     *
+     * @param key Key.
+     * @param value Buffer in which to put the binary data.
+     * @param vsize in: size of the buffer, out: actual size of the data.
+     *
+     * @return true if the key exists and the read succeeded, false otherwise.
+     */
     virtual bool loadRawData(const std::string& key, char* value, size_t* vsize) const = 0;
 
+    /**
+     * @brief Loads binary data associated with a particular key from the container.
+     * This function will look in the prefetcher if the object has been prefetched
+     * (or scheduled to be prefetched) and fall back to looking up in the underlying
+     * DataStore if it hasn't.
+     *
+     * @param prefetcher Prefetcher to look into first.
+     * @param key Key.
+     * @param buffer Buffer in which to put the binary data.
+     *
+     * @return true if the key exists and the read succeeded, false otherwise.
+     */
     virtual bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const = 0;
 
+    /**
+     * @brief Loads binary data associated with a particular key from the container.
+     * This function will look in the prefetcher if the object has been prefetched
+     * (or scheduled to be prefetched) and fall back to looking up in the underlying
+     * DataStore if it hasn't.
+     *
+     * @param key Key.
+     * @param value Buffer in which to put the binary data.
+     * @param vsize in: size of the buffer, out: size of the actual data.
+     *
+     * @return true if the key exists and the read succeeded, false otherwise.
+     */
     virtual bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const = 0;
 
     /**
@@ -122,7 +176,7 @@ class KeyValueContainer {
      * @param key Key to store.
      * @param value Value to store.
      *
-     * @return true if the key was found. false otherwise.
+     * @return a valid ProductID if the key was found, an invalid one otherwise.
      */
     template<typename K, typename V>
     ProductID store(const K& key, const V& value) {
@@ -137,18 +191,41 @@ class KeyValueContainer {
      * string must not have the "/", or "%" characters. The
      * type of the value must be serializable using Boost.
      *
+     * Note that since the WriteBatch will delay the operation,
+     * the returned ProductID is valid even though the operation
+     * may not ultimately succeed.
+     *
      * @tparam K type of the key.
      * @tparam V type of the value.
      * @param key Key to store.
      * @param value Value to store.
      *
-     * @return true if the key was found. false otherwise.
+     * @return a valid ProductID.
      */
     template<typename K, typename V>
     ProductID store(WriteBatch& batch, const K& key, const V& value) {
         return storeImpl(batch, key, value, std::is_pod<V>());
     }
 
+    /**
+     * @brief Stores a key/value pair into the WriteBatch.
+     * The type of the key should have operator<< available
+     * to stream it into a std::stringstream for the purpose
+     * of converting it into an std::string. The resulting
+     * string must not have the "/", or "%" characters. The
+     * type of the value must be serializable using Boost.
+     *
+     * Note that since the WriteBatch will delay the operation,
+     * the returned ProductID is valid even though the operation
+     * may not ultimately succeed.
+     *
+     * @tparam K type of the key.
+     * @tparam V type of the value.
+     * @param key Key to store.
+     * @param value Value to store.
+     *
+     * @return a valid ProductID.
+     */
     template<typename K, typename V>
     ProductID store(AsyncEngine& async, const K& key, const V& value) {
         return storeImpl(async, key, value, std::is_pod<V>());
@@ -212,11 +289,19 @@ class KeyValueContainer {
         return loadVectorImpl(key, value, std::is_pod<V>());
     }
 
+    /**
+     * @brief Version of load that will first look into the Prefetcher
+     * argument for the requested key.
+     */
     template<typename K, typename V>
     bool load(const Prefetcher& prefetcher, const K& key, V& value) const {
         return loadImpl(prefetcher, key, value, std::is_pod<V>());
     }
 
+    /**
+     * @brief Version of load for vectors, looking first into the
+     * Prefetcher argument for the requested key.
+     */
     template<typename K, typename V>
     bool load(const Prefetcher& prefetcher, const K& key, std::vector<V>& value) const {
         return loadVectorImpl(prefetcher, key, value, std::is_pod<V>());
@@ -312,6 +397,9 @@ class KeyValueContainer {
         return vsize == sizeof(value);
     }
 
+    /**
+     * @brief Implementation of the load function with a prefetcher.
+     */
     template<typename K, typename V>
     bool loadImpl(const Prefetcher& prefetcher, const K& key, V& value,
             const std::integral_constant<bool, true>&) const {
@@ -347,6 +435,9 @@ class KeyValueContainer {
         return true;
     }
 
+    /**
+     * @brief Implementation of the load function with a prefetcher.
+     */
     template<typename K, typename V>
     bool loadImpl(const Prefetcher& prefetcher, const K& key, V& value,
             const std::integral_constant<bool, false>&) const {
@@ -391,6 +482,9 @@ class KeyValueContainer {
         return true;
     }
 
+    /**
+     * @brief Implementation of the load function with a prefetcher.
+     */
     template<typename K, typename V>
     bool loadVectorImpl(const Prefetcher& prefetcher, const K& key, std::vector<V>& value,
             const std::integral_constant<bool, true>&) const {
@@ -440,6 +534,9 @@ class KeyValueContainer {
         return true;
     }
 
+    /**
+     * @brief Implementation of the load function with a prefetcher.
+     */
     template<typename K, typename V>
     bool loadVectorImpl(const Prefetcher& prefetcher, const K& key, std::vector<V>& value,
             const std::integral_constant<bool, false>&) const {

include/hepnos/Prefetchable.hpp

+/*
+ * (C) 2018 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
 #ifndef __HEPNOS_PREFETCHABLE_HPP
 #define __HEPNOS_PREFETCHABLE_HPP
 
@@ -7,6 +12,14 @@ namespace hepnos {
 
 class Prefetcher;
 
+/**
+ * @brief Prefetchable is a template class that can wrap a container
+ * (RunSet, Run, and SubRun) to automatically pass a Prefetcher object
+ * to their begin() functions. It is meant to be created by the
+ * operator() of the Prefetcher class.
+ *
+ * @tparam Container container type (RunSet, Run, and SubRun).
+ */
 template<typename Container>
 class Prefetchable {
 
@@ -14,24 +27,59 @@ class Prefetchable {
 
     public:
 
+    /**
+     * @brief Deleted copy constructor.
+     */
     Prefetchable(const Prefetchable& other) = delete;
+
+    /**
+     * @brief Default move-assignment operator.
+     */
     Prefetchable(Prefetchable&& other) = default;
+
+    /**
+     * @brief Deleted copy-assignment.
+     */
     Prefetchable& operator=(const Prefetchable& other) = delete;
+
+    /**
+     * @brief Deleted move-assignment.
+     */
     Prefetchable& operator=(Prefetchable&& other) = delete;
+
+    /**
+     * @brief Default destructor.
+     */
     ~Prefetchable() = default;
 
+    /**
+     * @brief Returns a prefetch-enabled iterator to the beginning
+     * of the container.
+     */
     typename Container::iterator begin() {
         return m_container.begin(m_prefetcher);
     }
 
+    /**
+     * @brief Returns a prefetch-enabled iterator to the end of the
+     * container.
+     */
     typename Container::iterator end() {
         return m_container.end();
     }
 
+    /**
+     * @brief Returns a prefetch-enable const iterator to the beginning
+     * of the container.
+     */
     typename Container::const_iterator begin() const {
         return m_container.begin(m_prefetcher);
     }
 
+    /**
+     * @brief Returns a prefetch-enabled const iterator to the end
+     * of the container.
+     */
     typename Container::const_iterator end() const {
         return m_container.end();
     }

include/hepnos/Prefetcher.hpp

+/*
+ * (C) 2018 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
 #ifndef __HEPNOS_PREFETCHER_HPP
 #define __HEPNOS_PREFETCHER_HPP
 
@@ -16,6 +21,10 @@ class Run;
 class SubRun;
 class Event;
 
+/**
+ * @brief The Prefetcher object will actively try to prefetch
+ * items from the underlying DataStore when using iterators.
+ */
 class Prefetcher {
 
     friend class RunSet;
@@ -26,28 +35,100 @@ class Prefetcher {
 
     public:
 
+    /**
+     * @brief Constructor.
+     *
+     * @param ds DataStore instance to prefetch from.
+     * @param cache_size maximum number of items that can be stored in the cache.
+     * @param batch_size how many items to prefetch at once.
+     */
     Prefetcher(const DataStore& ds,
                unsigned int cache_size=16,
                unsigned int batch_size=16);
-    Prefetcher(const DataStore& ds, const AsyncEngine& async, 
+
+    /**
+     * @brief Constructor using an AsyncEngine to prefetch in the background.
+     *
+     * @param async AsyncEngine to prefetch in the background.
+     * @param cache_size maximum number of items that can be stored in the cache.
+     * @param batch_size how many items to prefetch at once.
+     */
+    Prefetcher(const AsyncEngine& async, 
                unsigned int cache_size=16,
                unsigned int batch_size=16);
+
+    /**
+     * @brief Destructor.
+     */
     ~Prefetcher();
+
+    /**
+     * @brief Deleted copy constructor.
+     */
     Prefetcher(const Prefetcher&) = delete;
+
+    /**
+     * @brief Deleted move constructor.
+     */
     Prefetcher(Prefetcher&&) = delete;
+