Commit 33018afe authored by Matthieu Dorier's avatar Matthieu Dorier

added support for DataSets storing Runs

parent fc5472c0
......@@ -11,6 +11,8 @@ cmake_minimum_required (VERSION 3.0)
project (hepnos CXX)
enable_testing ()
add_definitions (-g)
# add our cmake module directory to the path
set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
"${CMAKE_CURRENT_SOURCE_DIR}/cmake")
......
......@@ -8,10 +8,14 @@
#include <boost/archive/text_iarchive.hpp>
#include <boost/serialization/string.hpp>
#include <hepnos/Exception.hpp>
#include <hepnos/RunNumber.hpp>
#include <hepnos/DataStore.hpp>
namespace hepnos {
class RunSet;
class Run;
/**
* @brief The DataSet class represents a handle to a named dataset
* stored either at the root of an HEPnOS DataStore service, or within
......@@ -21,6 +25,7 @@ namespace hepnos {
class DataSet {
friend class DataStore;
friend class RunSet;
private:
......@@ -258,6 +263,19 @@ class DataSet {
*/
DataSet createDataSet(const std::string& name);
/**
* @brief Creates a run with a given run number inside the DataSet.
* A Run object pointing to the created run is returned.
* If a run with the same number exists in this DataSet, the run
* is not created by a Run object pointing to the existing one is
* returned instead.
*
* @param runNumber Run number of the run to create.
*
* @return A Run instance pointing to the created run.
*/
Run createRun(const RunNumber& runNumber);
typedef DataStore::const_iterator const_iterator;
typedef DataStore::iterator iterator;
......@@ -309,12 +327,29 @@ class DataSet {
/**
* @brief Returns an iterator referring to the end of the DataSet.
* The DataSet pointed to by this iterator is not valid (that is,
* `end()->valid()` return `false`).
* `end()->valid()` returns `false`).
*
* @return an iterator referring to the end of the DataSet.
*/
iterator end();
/**
* @brief Returns a const_iterator referring to the first DataSet
* in this DataSet.
*
* @return a const_iterator referring to the first DataSet in this DataSet.
*/
const_iterator begin() const;
/**
* @brief Returns a const_iterator referring to the end of the DataSet.
* The DataSet pointed to by this iterator is not valid (that is,
* `end()->valid()` returns `false`).
*
* @return a const_iterator referring to the end of the DataSet.
*/
const_iterator end() const;
/**
* @brief Returns a const_iterator referring to the first DataSet
* in this DataSet.
......@@ -382,7 +417,31 @@ class DataSet {
*/
const_iterator upper_bound(const std::string& ub) const;
/**
* @brief Returns a reference to the RunSet associated with this DataSet.
*
* @return a reference to the RunSet associated with this DataSet.
*/
RunSet& runs();
/**
* @brief Returns a reference to the RunSet associated with this DataSet.
*
* @return a reference to the RunSet associated with this DataSet.
*/
const RunSet& runs() const;
/**
* @brief Accesses an existing run using the ()
* operator. If no run corresponds to the provided run number,
* the function returns a Run instance d such that
* r.valid() is false.
*
* @param runNumber Number of the run to retrieve.
*
* @return a Run corresponding to the provided run number.
*/
Run operator()(const RunNumber& runNumber) const;
};
}
......
......@@ -8,6 +8,9 @@
namespace hepnos {
class DataSet;
class RunSet;
class Run;
class SubRun;
/**
* The DataStore class is the main handle referencing an HEPnOS service.
......@@ -16,6 +19,9 @@ class DataSet;
class DataStore {
friend class DataSet;
friend class RunSet;
friend class Run;
friend class SubRun;
public:
......@@ -120,12 +126,29 @@ class DataStore {
/**
* @brief Returns an iterator referring to the end of the DataStore.
* The DataSet pointed to by this iterator is not valid (that is,
* `end()->valid()` return `false`).
* `end()->valid()` returns `false`).
*
* @return an iterator referring to the end of the DataStore.
*/
iterator end();
/**
* @brief Returns a const_iterator referring to the first DataSet
* in the DataStore.
*
* @return a const_iterator referring to the first DataSet in the DataStore.
*/
const_iterator begin() const;
/**
* @brief Returns a const_iterator referring to the end of the DataStore.
* The DataSet pointed to by this iterator is not valid (that is,
* `end()->valid()` returns `false`).
*
* @return a const_iterator referring to the end of the DataStore.
*/
const_iterator end() const;
/**
* @brief Returns a const_iterator referring to the first DataSet
* in the DataStore.
......
#ifndef __HEPNOS_RUN_H
#define __HEPNOS_RUN_H
#include <memory>
#include <string>
#include <hepnos/DataStore.hpp>
#include <hepnos/RunNumber.hpp>
namespace hepnos {
class RunSet;
class Run {
private:
friend class RunSet;
friend class DataSet;
class Impl;
std::unique_ptr<Impl> m_impl;
Run(DataStore* datastore, uint8_t level, const std::string& container, const RunNumber& run);
public:
Run();
Run(const Run&);
Run(Run&&);
Run& operator=(const Run&);
Run& operator=(Run&&);
~Run();
Run next() const;
bool valid() const;
bool storeRawData(const std::string& key, const std::vector<char>& buffer);
bool loadRawData(const std::string& key, std::vector<char>& buffer) const;
bool operator==(const Run& other) const;
bool operator!=(const Run& other) const;
const RunNumber& number() const;
const std::string& container() const;
};
}
#endif
#ifndef __HEPNOS_RUN_NUMBER_H
#define __HEPNOS_RUN_NUMBER_H
#include <cstdint>
namespace hepnos {
typedef std::uint64_t RunNumber;
}
#endif
This diff is collapsed.
# list of source files
set(hepnos-src DataStore.cpp DataSet.cpp)
set(hepnos-src DataStore.cpp DataSet.cpp RunSet.cpp Run.cpp)
# load package helper for generating cmake CONFIG packages
include (CMakePackageConfigHelpers)
......
#include "hepnos/DataSet.hpp"
#include "hepnos/Run.hpp"
#include "hepnos/RunSet.hpp"
#include "private/RunImpl.hpp"
#include "private/DataSetImpl.hpp"
#include "private/DataStoreImpl.hpp"
namespace hepnos {
class DataSet::Impl {
public:
DataStore* m_datastore;
uint8_t m_level;
std::string m_container;
std::string m_name;
Impl(DataStore* ds, uint8_t level, const std::string& container, const std::string& name)
: m_datastore(ds)
, m_level(level)
, m_container(container)
, m_name(name) {}
};
DataSet::DataSet()
: m_impl(std::make_unique<DataSet::Impl>(nullptr, 0, "", "")) {}
: m_impl(std::make_unique<DataSet::Impl>(this, nullptr, 0, "", "")) {}
DataSet::DataSet(DataStore* ds, uint8_t level, const std::string& fullname)
: m_impl(std::make_unique<DataSet::Impl>(ds, level, "", "")) {
: m_impl(std::make_unique<DataSet::Impl>(this, ds, level, "", "")) {
size_t p = fullname.find_last_of('/');
if(p == std::string::npos) {
m_impl->m_name = fullname;
......@@ -34,16 +22,24 @@ DataSet::DataSet(DataStore* ds, uint8_t level, const std::string& fullname)
}
DataSet::DataSet(DataStore* ds, uint8_t level, const std::string& container, const std::string& name)
: m_impl(std::make_unique<DataSet::Impl>(ds, level, container, name)) {}
: m_impl(std::make_unique<DataSet::Impl>(this, ds, level, container, name)) {}
DataSet::DataSet(const DataSet& other)
: m_impl(std::make_unique<DataSet::Impl>(*other.m_impl)) {}
: m_impl(std::make_unique<DataSet::Impl>(
this, other.m_impl->m_datastore,
other.m_impl->m_level,
other.m_impl->m_container,
other.m_impl->m_name)) {}
DataSet::DataSet(DataSet&&) = default;
DataSet& DataSet::operator=(const DataSet& other) {
if(this == &other) return *this;
m_impl = std::make_unique<DataSet::Impl>(*other.m_impl);
m_impl = std::make_unique<DataSet::Impl>(this,
other.m_impl->m_datastore,
other.m_impl->m_level,
other.m_impl->m_container,
other.m_impl->m_name);
return *this;
}
......@@ -118,11 +114,23 @@ DataSet DataSet::createDataSet(const std::string& name) {
return DataSet(m_impl->m_datastore, m_impl->m_level+1, parent, name);
}
Run DataSet::createRun(const RunNumber& runNumber) {
std::string parent = fullname();
std::string runStr = Run::Impl::makeKeyStringFromRunNumber(runNumber);
m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, runStr, std::vector<char>());
return Run(m_impl->m_datastore, m_impl->m_level+1, parent, runNumber);
}
DataSet DataSet::operator[](const std::string& datasetName) const {
auto it = find(datasetName);
return std::move(*it);
}
Run DataSet::operator()(const RunNumber& runNumber) const {
auto it = runs().find(runNumber);
return std::move(*it);
}
DataSet::iterator DataSet::find(const std::string& datasetName) {
int ret;
if(datasetName.find('/') != std::string::npos) {
......@@ -153,6 +161,14 @@ DataSet::iterator DataSet::end() {
return m_impl->m_datastore->end();
}
DataSet::const_iterator DataSet::begin() const {
return const_iterator(const_cast<DataSet*>(this)->begin());
}
DataSet::const_iterator DataSet::end() const {
return m_impl->m_datastore->cend();
}
DataSet::const_iterator DataSet::cbegin() const {
return const_iterator(const_cast<DataSet*>(this)->begin());
}
......@@ -194,4 +210,12 @@ DataSet::const_iterator DataSet::upper_bound(const std::string& ub) const {
return it;
}
RunSet& DataSet::runs() {
return m_impl->m_runset;
}
const RunSet& DataSet::runs() const {
return const_cast<DataSet*>(this)->runs();
}
}
......@@ -69,10 +69,19 @@ DataStore::iterator DataStore::begin() {
else return end();
}
DataStore::const_iterator DataStore::begin() const {
iterator it = const_cast<DataStore*>(this)->begin();
return const_iterator(std::move(it));
}
DataStore::iterator DataStore::end() {
return m_impl->m_end;
}
DataStore::const_iterator DataStore::end() const {
return m_impl->m_end;
}
DataStore::const_iterator DataStore::cbegin() const {
return const_cast<DataStore*>(this)->begin();
}
......
#include "hepnos/Run.hpp"
#include "private/RunImpl.hpp"
#include "private/DataStoreImpl.hpp"
namespace hepnos {
Run::Run()
: m_impl(std::make_unique<Run::Impl>(nullptr, 0, "", 0)) {}
Run::Run(DataStore* ds, uint8_t level, const std::string& container, const RunNumber& rn)
: m_impl(std::make_unique<Run::Impl>(ds, level, container, rn)) { }
Run::Run(const Run& other)
: m_impl(std::make_unique<Run::Impl>(*other.m_impl)) {}
Run::Run(Run&&) = default;
Run& Run::operator=(const Run& other) {
if(this == &other) return *this;
m_impl = std::make_unique<Run::Impl>(*other.m_impl);
return *this;
}
Run& Run::operator=(Run&&) = default;
Run::~Run() = default;
Run Run::next() const {
if(!valid()) return Run();
std::vector<std::string> keys;
size_t s = m_impl->m_datastore->m_impl->nextKeys(
m_impl->m_level, m_impl->m_container,
m_impl->makeKeyStringFromRunNumber(), keys, 1);
if(s == 0) return Run();
size_t i = m_impl->m_container.size()+1;
if(keys[0].size() <= i) return Run();
if(keys[0][i] != '%') return Run();
std::stringstream strRunNumber;
strRunNumber << std::hex << keys[0].substr(i+1);
RunNumber rn;
strRunNumber >> rn;
return Run(m_impl->m_datastore, m_impl->m_level, m_impl->m_container, rn);
}
bool Run::valid() const {
return m_impl && m_impl->m_datastore;
}
bool Run::storeRawData(const std::string& key, const std::vector<char>& buffer) {
if(!valid()) {
throw Exception("Calling store() on invalid Run");
}
// forward the call to the datastore's store function
return m_impl->m_datastore->m_impl->store(0, m_impl->fullpath(), key, buffer);
}
bool Run::loadRawData(const std::string& key, std::vector<char>& buffer) const {
if(!valid()) {
throw Exception("Calling load() on invalid Run");
}
// forward the call to the datastore's load function
return m_impl->m_datastore->m_impl->load(0, m_impl->fullpath(), key, buffer);
}
bool Run::operator==(const Run& other) const {
return m_impl->m_datastore == other.m_impl->m_datastore
&& m_impl->m_level == other.m_impl->m_level
&& m_impl->m_container == other.m_impl->m_container
&& m_impl->m_run_nr == other.m_impl->m_run_nr;
}
bool Run::operator!=(const Run& other) const {
return !(*this == other);
}
const RunNumber& Run::number() const {
return m_impl->m_run_nr;
}
const std::string& Run::container() const {
return m_impl->m_container;
}
/*
DataSet DataSet::createDataSet(const std::string& name) {
if(name.find('/') != std::string::npos) {
throw Exception("Invalid character '/' in dataset name");
}
std::string parent = fullname();
m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, name, std::vector<char>());
return DataSet(m_impl->m_datastore, m_impl->m_level+1, parent, name);
}
DataSet DataSet::operator[](const std::string& datasetName) const {
auto it = find(datasetName);
return std::move(*it);
}
DataSet::iterator DataSet::find(const std::string& datasetName) {
int ret;
if(datasetName.find('/') != std::string::npos) {
throw Exception("Invalid character '/' in dataset name");
}
std::vector<char> data;
std::string parent = fullname();
bool b = m_impl->m_datastore->m_impl->load(m_impl->m_level+1, parent, datasetName, data);
if(!b) {
return m_impl->m_datastore->end();
}
return iterator(DataSet(m_impl->m_datastore, m_impl->m_level+1, parent, datasetName));
}
DataSet::const_iterator DataSet::find(const std::string& datasetName) const {
iterator it = const_cast<DataSet*>(this)->find(datasetName);
return it;
}
DataSet::iterator DataSet::begin() {
DataSet ds(m_impl->m_datastore, m_impl->m_level+1, fullname(),"");
ds = ds.next();
if(ds.valid()) return iterator(ds);
else return end();
}
DataSet::iterator DataSet::end() {
return m_impl->m_datastore->end();
}
DataSet::const_iterator DataSet::cbegin() const {
return const_iterator(const_cast<DataSet*>(this)->begin());
}
DataSet::const_iterator DataSet::cend() const {
return m_impl->m_datastore->cend();
}
DataSet::iterator DataSet::lower_bound(const std::string& lb) {
std::string lb2 = lb;
size_t s = lb2.size();
lb2[s-1] -= 1; // sdskv_list_keys's start_key is exclusive
iterator it = find(lb2);
if(it != end()) {
// we found something before the specified lower bound
++it;
return it;
}
DataSet ds(m_impl->m_datastore, m_impl->m_level+1, fullname(), lb2);
ds = ds.next();
if(!ds.valid()) return end();
else return iterator(ds);
}
DataSet::const_iterator DataSet::lower_bound(const std::string& lb) const {
iterator it = const_cast<DataSet*>(this)->lower_bound(lb);
return it;
}
DataSet::iterator DataSet::upper_bound(const std::string& ub) {
DataSet ds(m_impl->m_datastore, m_impl->m_level+1, fullname(), ub);
ds = ds.next();
if(!ds.valid()) return end();
else return iterator(ds);
}
DataSet::const_iterator DataSet::upper_bound(const std::string& ub) const {
iterator it = const_cast<DataSet*>(this)->upper_bound(ub);
return it;
}
*/
}
#include <iomanip>
#include <sstream>
#include <string>
#include "hepnos/DataSet.hpp"
#include "hepnos/RunSet.hpp"
#include "private/DataSetImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/RunImpl.hpp"
#include "private/RunSetImpl.hpp"
namespace hepnos {
RunSet::RunSet(DataSet* ds)
: m_impl(std::make_unique<RunSet::Impl>(ds)) {}
RunSet::~RunSet() {}
RunSet::iterator RunSet::find(const RunNumber& runNumber) {
int ret;
std::vector<char> data;
std::string parent = m_impl->m_dataset->fullname();
std::string strNum = Run::Impl::makeKeyStringFromRunNumber(runNumber);
auto datastore = m_impl->m_dataset->m_impl->m_datastore;
auto level = m_impl->m_dataset->m_impl->m_level;
bool b = datastore->m_impl->load(level+1, parent, strNum, data);
if(!b) return end();
return iterator(Run(datastore, level+1, parent, runNumber));
}
RunSet::const_iterator RunSet::find(const RunNumber& runNumber) const {
iterator it = const_cast<RunSet*>(this)->find(runNumber);
return it;
}
RunSet::iterator RunSet::begin() {
auto it = find(0);
if(it != end()) return *it;
auto ds_level = m_impl->m_dataset->m_impl->m_level;
auto datastore = m_impl->m_dataset->m_impl->m_datastore;
std::string container = m_impl->m_dataset->fullname();
Run run(datastore, ds_level+1, container, 0);
run = run.next();
if(run.valid()) return iterator(run);
else return end();
}
RunSet::iterator RunSet::end() {
return m_impl->m_end;
}
RunSet::const_iterator RunSet::cbegin() const {
return const_iterator(const_cast<RunSet*>(this)->begin());
}
RunSet::const_iterator RunSet::cend() const {
return m_impl->m_end;
}
RunSet::const_iterator RunSet::begin() const {
return const_iterator(const_cast<RunSet*>(this)->begin());
}
RunSet::const_iterator RunSet::end() const {
return m_impl->m_end;
}
RunSet::iterator RunSet::lower_bound(const RunNumber& lb) {
if(lb == 0) {
auto it = find(0);
if(it != end()) {
return it;
} else {
Run run(m_impl->m_dataset->m_impl->m_datastore,
m_impl->m_dataset->m_impl->m_level+1,
m_impl->m_dataset->fullname(), 0);
run = run.next();
if(!run.valid()) return end();
else return iterator(run);
}
} else {
auto it = find(lb-1);
if(it != end()) {
++it;
return it;
}
Run run(m_impl->m_dataset->m_impl->m_datastore,
m_impl->m_dataset->m_impl->m_level+1,
m_impl->m_dataset->fullname(), lb-1);
run = run.next();
if(!run.valid()) return end();
else return iterator(run);
}
}
RunSet::const_iterator RunSet::lower_bound(const RunNumber& lb) const {
iterator it = const_cast<RunSet*>(this)->lower_bound(lb);
return it;
}
RunSet::iterator RunSet::upper_bound(const RunNumber& ub) {
Run run(m_impl->m_dataset->m_impl->m_datastore,
m_impl->m_dataset->m_impl->m_level+1,
m_impl->m_dataset->fullname(), ub);
run = run.next();
if(!run.valid()) return end();
else return iterator(run);
}
RunSet::const_iterator RunSet::upper_bound(const RunNumber& ub) const {
iterator it = const_cast<RunSet*>(this)->upper_bound(ub);
return it;
}
class RunSet::const_iterator::Impl {
public:
Run m_current_run;
Impl()
: m_current_run()
{}
Impl(const Run& run)
: m_current_run(run)
{}
Impl(Run&& run)
: m_current_run(std::move(run))
{}
Impl(const Impl& other)
: m_current_run(other.m_current_run)
{}
bool operator==(const Impl& other) const {
return m_current_run == other.m_current_run;
}
};
RunSet::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}
RunSet::const_iterator::const_iterator(const Run& run)
: m_impl(std::make_unique<Impl>(run)) {}
RunSet::const_iterator::const_iterator(Run&& run)
: m_impl(std::make_unique<Impl>(std::move(run))) {}
RunSet::const_iterator::~const_iterator() {}
RunSet::const_iterator::const_iterator(const RunSet::const_iterator& other)
: m_impl(std::make_unique<Impl>(*other.m_impl)) {}
RunSet::const_iterator::const_iterator(RunSet::const_iterator&& other)
: m_impl(std::move(other.m_impl)) {}
RunSet::const_iterator& RunSet::const_iterator::operator=(const RunSet::const_iterator& other) {
if(&other == this) return *this;
m_impl = std::make_unique<Impl>(*other.m_impl);
return *this;
}
RunSet::const_iterator& RunSet::const_iterator::operator=(RunSet::const_iterator&& other) {
if(&other == this) return *this;
m_impl = std::move(other.m_impl);
return *this;
}
RunSet::const_iterator::self_type RunSet::const_iterator::operator++() {
if(!m_impl) {
throw Exception("Trying to increment an invalid iterator");
}
m_impl->m_current_run = m_impl->m_current_run.next();
return *this;
}
RunSet::const_iterator::self_type RunSet::const_iterator::operator++(int) {
const_iterator copy = *this;
++(*this);
return copy;
}
const RunSet::const_iterator::reference RunSet::const_iterator::operator*() {
if(!m_impl) {
throw Exception("Trying to dereference an invalid iterator");
}
return m_impl->m_current_run;
}
<