Skip to content

H
HEPnOS
Commit d4c1d4a5 authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files
Options

better comments

parent 9df9a9bd
Hide whitespace changes
Inline Side-by-side
Showing with 874 additions and 92 deletions
include/hepnos.hpp
View file @ d4c1d4a5
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_H
#define __HEPNOS_H
......
include/hepnos/AsyncEngine.hpp
View file @ d4c1d4a5
......@@ -22,6 +22,16 @@ class Prefetcher;
class PrefetcherImpl;
class AsyncEngineImpl;
/**
* @brief The AsyncEngine class uses Argobots to provide a set
* of execution streams (ES) to perform operations in the background.
* This object may then be used by Prefetcher or WriteBatch
* instances to perform their RPCs in the background.
* The AsyncEngine may be initialized with 0 ES, in which case
* the communications will still happen in the background using
* Mercury's non-blocking functions, but the rest of the processing
* will happen synchronously.
*/
class AsyncEngine {
friend class DataStore;
......@@ -39,14 +49,72 @@ class AsyncEngine {
public:
/**
* @brief Constructor.
*
* @param ds DataStore instance.
* @param num_threads Number of execution streams (background threads) to start.
*/
AsyncEngine(DataStore& ds, size_t num_threads=0);
/**
* @brief Destructor.
*/
~AsyncEngine() = default;
/**
* @brief Copy constructor. A copy of an existing
* AsyncEngine will share the same pool of threads as the
* instance it was copied from.
*
* @param AsyncEngine AsyncEngine instance to copy.
*/
AsyncEngine(const AsyncEngine&) = default;
/**
* @brief Copy-assignment operator. A copy of
* an existing AsyncEngine will share the same pool of
* threads as the instance it was copied from.
*
* @param AsyncEngine AsyncEngine to copy.
*
* @return The assigned-to instance.
*/
AsyncEngine& operator=(const AsyncEngine&) = default;
/**
* @brief Move constructor. The thread pool will be
* moved to the new AsyncEngine instance, leaving the
* moved-from instance in an invalid state.
*
* @param AsyncEngine AsyncEngine to move from.
*/
AsyncEngine(AsyncEngine&&) = default;
/**
* @brief Move-assignment operator. The thread pool
* will be moved to the new moved-to AsyncEngine
* instance, leaving the moved-from instance in an
* invalid state.
*
* @param AsyncEngine AsyncEngine to move from.
*
* @return The moved-to instance.
*/
AsyncEngine& operator=(AsyncEngine&&) = default;
/**
* @brief Blocks until the threads in this AsyncEngine
* don't have any more work to do.
*/
void wait();
/**
* @brief Returns the list of errors that occured
* when running the asynchronous work.
*
* @return List of error messages.
*/
const std::vector<std::string>& errors() const;
};
......
include/hepnos/DataSet.hpp
View file @ d4c1d4a5
......@@ -32,9 +32,6 @@ class DataSet : public KeyValueContainer {
private:
/**
* @brief Implementation class (used for the Pimpl idiom).
*/
std::shared_ptr<DataSetImpl> m_impl; /*!< Pointer to implementation. */
/**
......@@ -137,34 +134,38 @@ class DataSet : public KeyValueContainer {
bool valid() const;
/**
* @brief Stores binary data associated with a particular key into this DataSet.
* This function will return true if the key did not already exist and the
* write succeeded. It will return false otherwise.
*
* @param key Key.
* @param buffer Binary data to insert.
*
* @return a valid ProductID if the key did not already exist and the write succeeded,
* an invalid one otherwise.
* @see KeyValueContainer::storeRawData()
*/
ProductID storeRawData(const std::string& key, const char* value, size_t vsize) override;
/**
* @see KeyValueContainer::storeRawData()
*/
ProductID storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) override;
/**
* @see KeyValueContainer::storeRawData()
*/
ProductID storeRawData(AsyncEngine& engine, const std::string& key, const char* value, size_t vsize) override;
/**
* @brief Loads binary data associated with a particular key from the DataSet.
* This function will return true if the key exists and the read succeeded.
* It will return false otherwise.
*
* @param key Key.
* @param buffer Buffer in which to put the binary data.
*
* @return true if the key exists and the read succeeded,
* false otherwise.
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const std::string& key, std::string& value) const override;
/**
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
/**
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& value) const override;
/**
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
/**
......@@ -213,7 +214,31 @@ class DataSet : public KeyValueContainer {
* @return A Run instance pointing to the created run.
*/
Run createRun(const RunNumber& runNumber);
/**
* @brief Creates a Run asynchronously.
* Note that even though the returned Run object is valid, since
* the actual creation operation will happen in the background,
* there is no guarantee that this operation will succeed.
*
* @param async AsyncEngine to use to make the operation happen in the background.
* @param runNumber Run number.
*
* @return a valid Run object.
*/
Run createRun(AsyncEngine& async, const RunNumber& runNumber);
/**
* @brief Creates a Run by pushing the creation operation into
* a WriteBatch. Note that even though the returned Run object is valid,
* since the actual creation is delayed until the WriteBatch is flushed,
* there is no guarantee that this operation will ultimately succeed.
*
* @param batch WriteBatch in which to append the operation.
* @param runNumber Run number.
*
* @return a valid Run object.
*/
Run createRun(WriteBatch& batch, const RunNumber& runNumber);
/**
......
include/hepnos/DataStore.hpp
View file @ d4c1d4a5
......@@ -182,9 +182,6 @@ class DataStore {
private:
/**
* @brief Implementation of the class (using Pimpl idiom)
*/
std::shared_ptr<DataStoreImpl> m_impl; /*!< Pointer to implementation */
/**
......
include/hepnos/Demangle.hpp
View file @ d4c1d4a5
......@@ -11,12 +11,23 @@
namespace hepnos {
/**
* @brief Returns the name of a type.
*
* @tparam T Type from which to return the name.
*
* @return Name of type T.
*/
template<typename T>
std::string demangle() {
char const * name = typeid(T).name();
return boost::core::demangle(name);
}
/**
* @brief Returns the name of type, which example
* instance provided for type deduction.
*/
template<typename T>
std::string demangle(T&&) {
return demangle<T>();
......
include/hepnos/Event.hpp
View file @ d4c1d4a5
......@@ -24,6 +24,10 @@ struct EventDescriptor {
char data[EventDescriptorLength];
};
/**
* @brief The Event class is a KeyValueContainer contained in SubRuns
* and with no sub-container.
*/
class Event : public KeyValueContainer {
private:
......@@ -107,28 +111,38 @@ class Event : public KeyValueContainer {
bool valid() const;
/**
* @brief Stores raw key/value data in this Event.
*
* @param key Key
* @param value Value
*
* @return a valid ProductID if the key did not already exist, an invalid one otherwise.
* @see KeyValueContainer::storeRawData()
*/
ProductID storeRawData(const std::string& key, const char* value, size_t vsize) override;
/**
* @see KeyValueContainer::storeRawData()
*/
ProductID storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) override;
/**
* @see KeyValueContainer::storeRawData()
*/
ProductID storeRawData(AsyncEngine& async, const std::string& key, const char* value, size_t vsize) override;
/**
* @brief Loads raw key/value data from this Event.
*
* @param key Key
* @param value Buffer used to hold the value.
*
* @return true if the key exists, false otherwise.
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const std::string& key, std::string& value) const override;
/**
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const std::string& key, char* value, size_t* vsize) const override;
/**
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const override;
/**
* @see KeyValueContainer::loadRawData()
*/
bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const override;
/**
......
include/hepnos/Exception.hpp
View file @ d4c1d4a5
......@@ -11,17 +11,42 @@
namespace hepnos {
/**
* @brief Exception class thrown by HEPnOS functions on failure.
*/
class Exception : public std::exception
{
std::string m_msg;
public:
/**
* @brief Default constructor.
*/
Exception() = default;
/**
* @brief Constructor.
*
* @param msg Error message.
*/
Exception(const std::string& msg) : m_msg(msg){}
/**
* @brief Copy constructor.
*/
Exception(const Exception&) = default;
/**
* @brief Copy-assignment operator.
*/
Exception& operator=(const Exception&) = default;
/**
* @brief Returns the error message.
*
* @return the error message.
*/
virtual const char* what() const noexcept override
{
return m_msg.c_str();
......
include/hepnos/InputArchive.hpp
View file @ d4c1d4a5
......@@ -34,17 +34,34 @@ class InputArchive : public iarchive {
public:
/**
* @brief Constructor.
*
* @tparam Args Argument types necessary for the underlying boost iarchive.
* @param datastore DataStore.
* @param args Arguments necessary for the parent boost iarchive.
*/
template<typename ... Args>
InputArchive(DataStore&& datastore, Args&& ... args)
: iarchive(std::forward<Args>(args)..., boost::archive::archive_flags::no_header)
, m_datastore(std::move(datastore)) {}
/**
* @brief Constructor.
*
* @tparam Args Argument types necessary for the underlying boost iarchive.
* @param datastore DataStore.
* @param args Arguments necessary for the parent boost iarchive.
*/
template<typename ... Args>
InputArchive(const DataStore& datastore, Args&& ... args)
: iarchive(std::forward<Args>(args)..., boost::archive::archive_flags::no_header)
, m_datastore(datastore) {}
DataStore datastore() const {
/**
* @brief Returns the DataStore used when creating the InputArchive.
*/
const DataStore& datastore() const {
return m_datastore;
}
......
include/hepnos/ItemType.hpp
View file @ d4c1d4a5
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_ITEM_TYPE_HPP
#define __HEPNOS_ITEM_TYPE_HPP
namespace hepnos {
/**
* @brief Type of item (DataSet, Run, SubRun, Event, or Product).
*/
enum class ItemType : uint32_t {
DATASET,
RUN,
......
include/hepnos/KeyValueContainer.hpp
View file @ d4c1d4a5
......@@ -81,15 +81,34 @@ class KeyValueContainer {
* @brief Stores raw key/value data in a WriteBatch.
* This function is virtual and must be overloaded in the child class.
*
* Note since the WriteBatch is flushed later to the DataStore, the DataStore will
* not be able to check whether the product could be created or not. Hence the
* ProductID returned is valid but may not ultimately correspond to an actual
* Product in the DataStore, should the storage operation fail.
*
* @param batch Batch in which to write.
* @param key Key
* @param value Value pointer
* @param vsize Value size (in bytes)
*
* @return
* @return A valid ProductID.
*/
virtual ProductID storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) = 0;
/**
* @brief Stores binary data associated with a particular key using the AsyncEngine.
* Note since the AsyncEngine makes the operation happen later in the background,
* the DataStore will not be able to check whether the product could be created or not.
* Hence the ProductID returned is valid but may not ultimately correspond to an actual
* Product in the DataStore, should the storage operation fail.
*
* @param engine AsyncEngine to use to write asynchronously.
* @param key Key.
* @param value Binary data to write.
* @param vsize Size of the data (in bytes).
*
* @return a valid ProductID.
*/
virtual ProductID storeRawData(AsyncEngine& async, const std::string& key, const char* value, size_t vsize) = 0;
/**
......@@ -103,10 +122,45 @@ class KeyValueContainer {
*/
virtual bool loadRawData(const std::string& key, std::string& buffer) const = 0;
/**
* @brief Loads binary data associated with a particular key from the container.
* This function will return true if the key exists and the read succeeded.
* It will return false otherwise.
*
* @param key Key.
* @param value Buffer in which to put the binary data.
* @param vsize in: size of the buffer, out: actual size of the data.
*
* @return true if the key exists and the read succeeded, false otherwise.
*/
virtual bool loadRawData(const std::string& key, char* value, size_t* vsize) const = 0;
/**
* @brief Loads binary data associated with a particular key from the container.
* This function will look in the prefetcher if the object has been prefetched
* (or scheduled to be prefetched) and fall back to looking up in the underlying
* DataStore if it hasn't.
*
* @param prefetcher Prefetcher to look into first.
* @param key Key.
* @param buffer Buffer in which to put the binary data.
*
* @return true if the key exists and the read succeeded, false otherwise.
*/
virtual bool loadRawData(const Prefetcher& prefetcher, const std::string& key, std::string& buffer) const = 0;
/**
* @brief Loads binary data associated with a particular key from the container.
* This function will look in the prefetcher if the object has been prefetched
* (or scheduled to be prefetched) and fall back to looking up in the underlying
* DataStore if it hasn't.
*
* @param key Key.
* @param value Buffer in which to put the binary data.
* @param vsize in: size of the buffer, out: size of the actual data.
*
* @return true if the key exists and the read succeeded, false otherwise.
*/
virtual bool loadRawData(const Prefetcher& prefetcher, const std::string& key, char* value, size_t* vsize) const = 0;
/**
......@@ -122,7 +176,7 @@ class KeyValueContainer {
* @param key Key to store.
* @param value Value to store.
*
* @return true if the key was found. false otherwise.
* @return a valid ProductID if the key was found, an invalid one otherwise.
*/
template<typename K, typename V>
ProductID store(const K& key, const V& value) {
......@@ -137,18 +191,41 @@ class KeyValueContainer {
* string must not have the "/", or "%" characters. The
* type of the value must be serializable using Boost.
*
* Note that since the WriteBatch will delay the operation,
* the returned ProductID is valid even though the operation
* may not ultimately succeed.
*
* @tparam K type of the key.
* @tparam V type of the value.
* @param key Key to store.
* @param value Value to store.
*
* @return true if the key was found. false otherwise.
* @return a valid ProductID.
*/
template<typename K, typename V>
ProductID store(WriteBatch& batch, const K& key, const V& value) {
return storeImpl(batch, key, value, std::is_pod<V>());
}
/**
* @brief Stores a key/value pair into the WriteBatch.
* The type of the key should have operator<< available
* to stream it into a std::stringstream for the purpose
* of converting it into an std::string. The resulting
* string must not have the "/", or "%" characters. The
* type of the value must be serializable using Boost.
*
* Note that since the WriteBatch will delay the operation,
* the returned ProductID is valid even though the operation
* may not ultimately succeed.
*
* @tparam K type of the key.
* @tparam V type of the value.
* @param key Key to store.
* @param value Value to store.
*
* @return a valid ProductID.
*/
template<typename K, typename V>
ProductID store(AsyncEngine& async, const K& key, const V& value) {
return storeImpl(async, key, value, std::is_pod<V>());
......@@ -212,11 +289,19 @@ class KeyValueContainer {
return loadVectorImpl(key, value, std::is_pod<V>());
}
/**
* @brief Version of load that will first look into the Prefetcher
* argument for the requested key.
*/
template<typename K, typename V>
bool load(const Prefetcher& prefetcher, const K& key, V& value) const {
return loadImpl(prefetcher, key, value, std::is_pod<V>());
}
/**
* @brief Version of load for vectors, looking first into the
* Prefetcher argument for the requested key.
*/
template<typename K, typename V>
bool load(const Prefetcher& prefetcher, const K& key, std::vector<V>& value) const {
return loadVectorImpl(prefetcher, key, value, std::is_pod<V>());
......@@ -312,6 +397,9 @@ class KeyValueContainer {
return vsize == sizeof(value);
}
/**
* @brief Implementation of the load function with a prefetcher.
*/
template<typename K, typename V>
bool loadImpl(const Prefetcher& prefetcher, const K& key, V& value,
const std::integral_constant<bool, true>&) const {
......@@ -347,6 +435,9 @@ class KeyValueContainer {
return true;
}
/**
* @brief Implementation of the load function with a prefetcher.
*/
template<typename K, typename V>
bool loadImpl(const Prefetcher& prefetcher, const K& key, V& value,
const std::integral_constant<bool, false>&) const {
......@@ -391,6 +482,9 @@ class KeyValueContainer {
return true;
}
/**
* @brief Implementation of the load function with a prefetcher.
*/
template<typename K, typename V>
bool loadVectorImpl(const Prefetcher& prefetcher, const K& key, std::vector<V>& value,
const std::integral_constant<bool, true>&) const {
......@@ -440,6 +534,9 @@ class KeyValueContainer {
return true;
}
/**
* @brief Implementation of the load function with a prefetcher.
*/
template<typename K, typename V>
bool loadVectorImpl(const Prefetcher& prefetcher, const K& key, std::vector<V>& value,
const std::integral_constant<bool, false>&) const {
......
include/hepnos/Prefetchable.hpp
View file @ d4c1d4a5
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_PREFETCHABLE_HPP
#define __HEPNOS_PREFETCHABLE_HPP
......@@ -7,6 +12,14 @@ namespace hepnos {
class Prefetcher;
/**
* @brief Prefetchable is a template class that can wrap a container
* (RunSet, Run, and SubRun) to automatically pass a Prefetcher object
* to their begin() functions. It is meant to be created by the
* operator() of the Prefetcher class.
*
* @tparam Container container type (RunSet, Run, and SubRun).
*/
template<typename Container>
class Prefetchable {
......@@ -14,24 +27,59 @@ class Prefetchable {
public:
/**
* @brief Deleted copy constructor.
*/
Prefetchable(const Prefetchable& other) = delete;
/**
* @brief Default move-assignment operator.
*/
Prefetchable(Prefetchable&& other) = default;
/**
* @brief Deleted copy-assignment.
*/
Prefetchable& operator=(const Prefetchable& other) = delete;
/**
* @brief Deleted move-assignment.
*/
Prefetchable& operator=(Prefetchable&& other) = delete;
/**
* @brief Default destructor.
*/
~Prefetchable() = default;
/**
* @brief Returns a prefetch-enabled iterator to the beginning
* of the container.
*/
typename Container::iterator begin() {
return m_container.begin(m_prefetcher);
}
/**
* @brief Returns a prefetch-enabled iterator to the end of the
* container.
*/
typename Container::iterator end() {
return m_container.end();
}
/**
* @brief Returns a prefetch-enable const iterator to the beginning
* of the container.
*/
typename Container::const_iterator begin() const {
return m_container.begin(m_prefetcher);
}
/**
* @brief Returns a prefetch-enabled const iterator to the end
* of the container.
*/
typename Container::const_iterator end() const {
return m_container.end();
}
......
include/hepnos/Prefetcher.hpp
View file @ d4c1d4a5
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_PREFETCHER_HPP
#define __HEPNOS_PREFETCHER_HPP
......@@ -16,6 +21,10 @@ class Run;
class SubRun;
class Event;
/**
* @brief The Prefetcher object will actively try to prefetch
* items from the underlying DataStore when using iterators.
*/
class Prefetcher {
friend class RunSet;
......@@ -26,28 +35,100 @@ class Prefetcher {
public:
/**
* @brief Constructor.
*
* @param ds DataStore instance to prefetch from.
* @param cache_size maximum number of items that can be stored in the cache.
* @param batch_size how many items to prefetch at once.
*/
Prefetcher(const DataStore& ds,
unsigned int cache_size=16,
unsigned int batch_size=16);
Prefetcher(const DataStore& ds, const AsyncEngine& async,
/**
* @brief Constructor using an AsyncEngine to prefetch in the background.
*
* @param async AsyncEngine to prefetch in the background.
* @param cache_size maximum number of items that can be stored in the cache.
* @param batch_size how many items to prefetch at once.
*/
Prefetcher(const AsyncEngine& async,
unsigned int cache_size=16,
unsigned int batch_size=16);
/**
* @brief Destructor.
*/
~Prefetcher();
/**
* @brief Deleted copy constructor.
*/
Prefetcher(const Prefetcher&) = delete;
/**
* @brief Deleted move constructor.
*/
Prefetcher(Prefetcher&&) = delete;
/**