Commit 8116e5be authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

Merge branch 'dev-event-set'

parents 4efe19bc 210b7dc8
......@@ -7,6 +7,7 @@
#include <hepnos/Demangle.hpp>
#include <hepnos/Event.hpp>
#include <hepnos/EventNumber.hpp>
#include <hepnos/EventSet.hpp>
#include <hepnos/Exception.hpp>
#include <hepnos/KeyValueContainer.hpp>
#include <hepnos/Run.hpp>
......
......@@ -369,7 +369,7 @@ class DataSet : public KeyValueContainer {
/**
* @brief Returns the RunSet associated with this DataSet.
*
* @return a reference to the RunSet associated with this DataSet.
* @return the RunSet associated with this DataSet.
*/
RunSet runs() const;
......@@ -384,6 +384,17 @@ class DataSet : public KeyValueContainer {
* @return a Run corresponding to the provided run number.
*/
Run operator[](const RunNumber& runNumber) const;
/**
* @brief Returns an EventSet pointing to Events in
* the specified target. If target is -1, the EventSet
* will also iterate over targets.
*
* @param target Target index in which to find events.
*
* @return an EventSet associated with the DataSet.
*/
EventSet events(int target=-1) const;
};
class DataSet::const_iterator {
......
......@@ -9,6 +9,7 @@
#include <vector>
#include <string>
#include <memory>
#include <hepnos/ItemType.hpp>
namespace hepnos {
......@@ -167,6 +168,16 @@ class DataStore {
*/
void shutdown();
/**
* @brief Returns the number of underlying targets for the
* specified item type.
*
* @param type Item type.
*
* @return The number of targets for the item type.
*/
size_t numTargets(const ItemType& type) const;
private:
/**
......
......@@ -15,11 +15,15 @@
namespace hepnos {
class SubRun;
class EventSet;
class Event : public KeyValueContainer {
private:
friend class SubRun;
friend class EventSet;
std::shared_ptr<ItemImpl> m_impl;
......@@ -146,6 +150,12 @@ class Event : public KeyValueContainer {
*/
const EventNumber& number() const;
/**
* @brief Returns an intance of the enclosing SubRun.
*
* @return Parent SubRun.
*/
SubRun subrun() const;
};
}
......
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_EVENT_SET_H
#define __HEPNOS_EVENT_SET_H
#include <memory>
#include <hepnos/Exception.hpp>
#include <hepnos/DataStore.hpp>
#include <hepnos/EventNumber.hpp>
#include <hepnos/RunNumber.hpp>
#include <hepnos/SubRunNumber.hpp>
namespace hepnos {
class EventSetImpl;
/**
* @brief The EventSet class is a helper class to access Events
* stored down in a particular DataSet or Run (bypassing
* intermediate nesting levels).
*/
class EventSet {
friend class DataSet;
private:
/**
* @brief Implementation class (used for the Pimpl idiom).
*/
std::shared_ptr<EventSetImpl> m_impl; /*!< Pointer to implementation. */
/**
* @brief Constructor.
*/
EventSet(const std::shared_ptr<EventSetImpl>& impl);
EventSet(std::shared_ptr<EventSetImpl>&& impl);
public:
typedef Event value_type;
/**
* @brief Copy-constructor.
*
* @param other EventSet to copy.
*/
EventSet(const EventSet& other) = default;
/**
* @brief Move-constructor. Deleted.
*
* @param other EventSet to move.
*/
EventSet(EventSet&& other) = default;
/**
* @brief Copy-assignment operator.
*
* @param other EventSet to copy.
*
* @return this.
*/
EventSet& operator=(const EventSet& other) = default;
/**
* @brief Move-assignment operator. Deleted.
*
* @param other EventSet to move.
*
* @return this.
*/
EventSet& operator=(EventSet&& other) = default;
/**
* @brief Destructor.
*/
~EventSet() = default;
class const_iterator;
class iterator;
/**
* @brief Get the DataStore to which this EventSet belongs.
*/
DataStore datastore() const;
/**
* @brief Returns an iterator referring to the first Event
* in this EventSet.
*
* @return an iterator referring to the first Event in this EventSet.
*/
iterator begin();
/**
* @brief Returns an iterator referring to the end of the EventSet.
* The EventSet pointed to by this iterator is not valid (that is,
* `end()->valid()` returns `false`).
*
* @return an iterator referring to the end of the EventSet.
*/
iterator end();
/**
* @brief Returns a const_iterator referring to the first Event
* in this EventSet.
*
* @return an iterator referring to the first Event in this EventSet.
*/
const_iterator begin() const;
/**
* @brief Returns a const_iterator referring to the end of the EventSet.
* The EventSet pointed to by this iterator is not valid (that is,
* `end()->valid()` returns `false`).
*
* @return a const_iterator referring to the end of the EventSet.
*/
const_iterator end() const;
/**
* @brief Returns a const_iterator referring to the first Event
* in this EventSet.
*
* @return a const_iterator referring to the first Event in this EventSet.
*/
const_iterator cbegin() const;
/**
* @brief Returns a const_iterator referring to the end of the EventSet.
* The EventSet pointed to by this iterator is not valid (that is,
* `cend()->valid()` return `false`).
*
* @return a const_iterator referring to the end of the EventSet.
*/
const_iterator cend() const;
};
class EventSet::const_iterator {
friend class EventSet;
protected:
/**
* @brief Implementation of the class (using Pimpl idiom)
*/
class Impl;
std::unique_ptr<Impl> m_impl; /*!< Pointer to implementation */
const_iterator(std::unique_ptr<Impl>&& impl);
public:
/**
* @brief Constructor. Creates a const_iterator pointing
* to an invalid Event.
*/
const_iterator();
typedef const_iterator self_type;
typedef Event value_type;
typedef Event& reference;
typedef Event* pointer;
typedef int difference_type;
typedef std::forward_iterator_tag iterator_category;
/**
* @brief Destructor. This destructor is virtual because
* the iterator class inherits from const_iterator.
*/
virtual ~const_iterator();
/**
* @brief Copy-constructor.
*
* @param other const_iterator to copy.
*/
const_iterator(const const_iterator& other);
/**
* @brief Move-constructor.
*
* @param other const_iterator to move.
*/
const_iterator(const_iterator&& other);
/**
* @brief Copy-assignment operator.
*
* @param other const_iterator to copy.
*
* @return this.
*/
const_iterator& operator=(const const_iterator&);
/**
* @brief Move-assignment operator.
*
* @param other const_iterator to move.
*
* @return this.
*/
const_iterator& operator=(const_iterator&&);
/**
* @brief Increments the const_iterator, returning
* a copy of the iterator after incrementation.
*
* @return a copy of the iterator after incrementation.
*/
self_type operator++();
/**
* @brief Increments the const_iterator, returning
* a copy of the iterator before incrementation.
*
* @return a copy of the iterator after incrementation.
*/
self_type operator++(int);
/**
* @brief Dereference operator. Returns a const reference
* to the DataSet this const_iterator points to.
*
* @return a const reference to the DataSet this
* const_iterator points to.
*/
const reference operator*();
/**
* @brief Returns a const pointer to the DataSet this
* const_iterator points to.
*
* @return a const pointer to the DataSet this
* const_iterator points to.
*/
const pointer operator->();
/**
* @brief Compares two const_iterators. The two const_iterators
* are equal if they point to the same DataSet or if both
* correspond to EventSet::cend().
*
* @param rhs const_iterator to compare with.
*
* @return true if the two const_iterators are equal, false otherwise.
*/
bool operator==(const self_type& rhs) const;
/**
* @brief Compares two const_iterators.
*
* @param rhs const_iterator to compare with.
*
* @return true if the two const_iterators are different, false atherwise.
*/
bool operator!=(const self_type& rhs) const;
};
class EventSet::iterator : public EventSet::const_iterator {
friend class EventSet;
private:
iterator(std::unique_ptr<EventSet::const_iterator::Impl>&& impl);
public:
/**
* @brief Constructor. Builds an iterator pointing to an
* invalid Event.
*/
iterator();
typedef iterator self_type;
typedef Event value_type;
typedef Event& reference;
typedef Event* pointer;
typedef int difference_type;
typedef std::forward_iterator_tag iterator_category;
/**
* @brief Destructor.
*/
~iterator();
/**
* @brief Copy constructor.
*
* @param other iterator to copy.
*/
iterator(const iterator& other);
/**
* @brief Move constructor.
*
* @param other iterator to move.
*/
iterator(iterator&& other);
/**
* @brief Copy-assignment operator.
*
* @param other iterator to copy.
*
* @return this.
*/
iterator& operator=(const iterator& other);
/**
* @brief Move-assignment operator.
*
* @param other iterator to move.
*
* @return this.
*/
iterator& operator=(iterator&& other);
/**
* @brief Dereference operator. Returns a reference
* to the DataSet this iterator points to.
*
* @return A reference to the DataSet this iterator
* points to.
*/
reference operator*();
/**
* @brief Returns a pointer to the DataSet this iterator
* points to.
*
* @return A pointer to the DataSet this iterator points to.
*/
pointer operator->();
};
}
#endif
#ifndef __HEPNOS_ITEM_TYPE_HPP
#define __HEPNOS_ITEM_TYPE_HPP
namespace hepnos {
enum class ItemType : uint32_t {
DATASET,
RUN,
SUBRUN,
EVENT,
PRODUCT
};
}
#endif
......@@ -22,6 +22,7 @@ class Run : public KeyValueContainer {
private:
friend class SubRun;
friend class RunSet;
friend class DataSet;
......
......@@ -20,6 +20,7 @@ class SubRun : public KeyValueContainer {
private:
friend class Event;
friend class Run;
std::shared_ptr<ItemImpl> m_impl;
......@@ -149,6 +150,11 @@ class SubRun : public KeyValueContainer {
*/
const SubRunNumber& number() const;
/**
* @brief Returns an instance of the enclosing Run.
*/
Run run() const;
class const_iterator;
class iterator;
......
......@@ -96,10 +96,16 @@ class AsyncEngineImpl {
id.run = run_number;
id.subrun = subrun_number;
id.event = event_number;
ItemType type = ItemType::RUN;
if(subrun_number != InvalidSubRunNumber) {
type = ItemType::SUBRUN;
if(event_number != InvalidEventNumber)
type = ItemType::EVENT;
}
// make a thread that will store the data
m_pool.make_thread([this, id, ds=m_datastore]() {
m_pool.make_thread([this, id, type, ds=m_datastore]() {
// locate db
auto& db = ds->locateItemDb(id);
auto& db = ds->locateItemDb(type, id);
try {
db.put(&id, sizeof(id), nullptr, 0);
} catch(sdskv::exception& ex) {
......
......@@ -7,7 +7,8 @@ set(hepnos-src DataStore.cpp
Event.cpp
UUID.cpp
WriteBatch.cpp
AsyncEngine.cpp)
AsyncEngine.cpp
EventSet.cpp)
set(hepnos-service-src service/HEPnOSService.cpp
service/ServiceConfig.cpp
......
......@@ -9,6 +9,7 @@
#include "hepnos/AsyncEngine.hpp"
#include "ItemImpl.hpp"
#include "DataSetImpl.hpp"
#include "EventSetImpl.hpp"
#include "DataStoreImpl.hpp"
#include "AsyncEngineImpl.hpp"
#include "WriteBatchImpl.hpp"
......@@ -349,6 +350,24 @@ RunSet DataSet::runs() const {
return RunSet(m_impl);
}
EventSet DataSet::events(int target) const {
if(!valid()) {
throw Exception("Calling DataSet member function on an invalid DataSet");
}
auto numTargets = m_impl->m_datastore->numTargets(ItemType::EVENT);
if(target >= (int)numTargets) {
throw Exception(std::string("Invalid target number ")
+std::to_string(target)
+" for EventSet (>= "
+std::to_string(numTargets)+")");
}
if(target >= 0)
return EventSet(std::make_shared<EventSetImpl>(*m_impl, target));
else {
return EventSet(std::make_shared<EventSetImpl>(*m_impl, 0, numTargets));
}
}
////////////////////////////////////////////////////////////////////////////////////////////
// DataSet::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
......
......@@ -73,5 +73,12 @@ bool DataStore::loadRawProduct(const ProductID& productID, char* data, size_t* s
return m_impl->loadRawProduct(productID, data, size);
}
size_t DataStore::numTargets(const ItemType& type) const {
if(!m_impl) {
throw Exception("Calling DataStore member function on an invalid DataStore object");
}
return m_impl->numTargets(type);
}
}
......@@ -154,6 +154,22 @@ class DataStoreImpl {
if(m_mid) margo_finalize(m_mid);
}
size_t numTargets(const ItemType& type) const {
switch(type) {
case ItemType::DATASET:
return m_dataset_dbs.dbs.size();
case ItemType::RUN:
return m_run_dbs.dbs.size();
case ItemType::SUBRUN:
return m_subrun_dbs.dbs.size();
case ItemType::EVENT:
return m_event_dbs.dbs.size();
case ItemType::PRODUCT:
return m_product_dbs.dbs.size();
}
return 0;
}
private:
static void checkConfig(YAML::Node& config) {
......@@ -478,15 +494,20 @@ class DataStoreImpl {
// Access functions for numbered items (Runs, SubRuns, and Events)
///////////////////////////////////////////////////////////////////////////
const sdskv::database& locateItemDb(const ItemDescriptor& id) const {
const sdskv::database& locateItemDb(const ItemType& type, const ItemDescriptor& id, int target=-1) const {
long unsigned db_idx = 0;
if(target >= 0) {
if(type == ItemType::RUN) return m_run_dbs.dbs[target];
if(type == ItemType::SUBRUN) return m_subrun_dbs.dbs[target];
if(type == ItemType::EVENT) return m_event_dbs.dbs[target];
}
uint64_t hash;
size_t prime = 1099511628211ULL;
hash = id.dataset.hash();
if(id.subrun == InvalidSubRunNumber) { // we are locating a Run
if(type == ItemType::RUN) { // we are locating a Run
ch_placement_find_closest(m_run_dbs.chi, hash, 1, &db_idx);
return m_run_dbs.dbs[db_idx];
} else if(id.event == InvalidEventNumber) { // we are locating a SubRun
} else if(type == ItemType::SUBRUN) { // we are locating a SubRun
hash *= prime;
hash = hash ^ id.run;
ch_placement_find_closest(m_subrun_dbs.chi, hash, 1, &db_idx);
......@@ -504,13 +525,17 @@ class DataStoreImpl {
* maxRuns shared_ptr to RunImpl coming after the
* current run. Returns the number of Runs read.
*/
size_t nextItems(const std::shared_ptr<ItemImpl>& current,
size_t nextItems(
const ItemType& item_type,
const ItemType& prefix_type,
const std::shared_ptr<ItemImpl>& current,
std::vector<std::shared_ptr<ItemImpl>>& result,
size_t maxItems) const {
size_t maxItems,
int target=-1) const {
int ret;
result.resize(0);
const ItemDescriptor& start_key = current->m_descriptor;
auto& db = locateItemDb(start_key);
auto& db = locateItemDb(item_type, start_key, target);
// ignore keys that don't have the same uuid