Skip to content

H
HEPnOS
Commit 651208a7 authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files
Options

started implementation of real mochi-based storage system

parent 98e7aee4
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 643 additions and 676 deletions
CMakeLists.txt
View file @ 651208a7
......@@ -34,9 +34,16 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
# packages we depend on
include (xpkg-import)
find_package (mercury CONFIG REQUIRED)
find_package (Boost REQUIRED COMPONENTS serialization filesystem)
find_package (Boost REQUIRED COMPONENTS serialization system filesystem)
include_directories(${Boost_INCLUDE_DIRS})
xpkg_import_module (margo REQUIRED margo)
xpkg_import_module (sdskv-client REQUIRED sdskv-client)
xpkg_import_module (bake-client REQUIRED bake-client)
xpkg_import_module (sdskv-server REQUIRED sdskv-server)
xpkg_import_module (bake-server REQUIRED bake-server)
xpkg_import_module (ch-placement REQUIRED ch-placement)
find_package (yaml-cpp REQUIRED)
add_subdirectory (src)
add_subdirectory (test)
#add_subdirectory (bin)
include/hepnos/DataSet.hpp 0 → 100644
View file @ 651208a7
#ifndef __HEPNOS_DATA_SET_H
#define __HEPNOS_DATA_SET_H
#include <hepnos/DataStore.hpp>
namespace hepnos {
class DataSet {
friend class DataStore;
private:
DataSet(DataStore& ds, uint8_t level, const std::string& name);
DataStore& m_datastore;
uint8_t m_level;
std::string m_name;
};
}
#endif
include/hepnos/DataStore.hpp 0 → 100644
View file @ 651208a7
#ifndef __HEPNOS_DATA_STORE_H
#define __HEPNOS_DATA_STORE_H
#include <string>
#include <memory>
namespace hepnos {
class DataSet;
class DataStore {
public:
DataStore(const std::string& configFile);
DataStore(const DataStore&) = delete;
DataStore(DataStore&&);
DataStore& operator=(const DataStore&) = delete;
DataStore& operator=(DataStore&&);
~DataStore();
class iterator;
class const_iterator;
iterator find(const std::string& datasetName);
inline const_iterator find(const std::string& datasetName) const;
iterator begin();
iterator end();
const_iterator cbegin() const;
const_iterator cend() const;
iterator lower_bound(const std::string& lb);
const_iterator lower_bound(const std::string& lb) const;
iterator upper_bound(const std::string& ub);
const_iterator upper_bound(const std::string& ub) const;
DataSet createDataSet(const std::string& name);
private:
class Impl;
std::unique_ptr<Impl> m_impl;
};
class DataStore::const_iterator {
friend class DataStore::Impl;
friend class DataStore;
protected:
DataStore* m_datastore;
const_iterator(DataStore& ds);
public:
typedef const_iterator self_type;
typedef DataSet value_type;
typedef DataSet& reference;
typedef DataSet* pointer;
typedef int difference_type;
typedef std::forward_iterator_tag iterator_category;
virtual ~const_iterator();
const_iterator(const const_iterator&);
const_iterator(const_iterator&&);
const_iterator& operator=(const const_iterator&);
const_iterator& operator=(const_iterator&&);
self_type operator++();
self_type operator++(int);
const reference operator*();
const pointer operator->();
bool operator==(const self_type& rhs) const;
bool operator!=(const self_type& rhs) const;
};
class DataStore::iterator : public DataStore::const_iterator {
friend class DataStore::Impl;
friend class DataStore;
private:
iterator(DataStore& ds);
public:
typedef iterator self_type;
typedef DataSet value_type;
typedef DataSet& reference;
typedef DataSet* pointer;
typedef int difference_type;
typedef std::forward_iterator_tag iterator_category;
~iterator();
iterator(const iterator&);
iterator(iterator&&);
iterator& operator=(const iterator&);
iterator& operator=(iterator&&);
reference operator*();
pointer operator->();
};
}
#endif
include/hepnos/Exception.hpp 0 → 100644
View file @ 651208a7
#ifndef __HEPNOS_EXCEPTION_H
#define __HEPNOS_EXCEPTION_H
#include <string>
#include <exception>
namespace hepnos {
class Exception : public std::exception
{
std::string m_msg;
public:
Exception(const std::string& msg) : m_msg(msg){}
virtual const char* what() const noexcept override
{
return m_msg.c_str();
}
};
}
#endif
include/hepnos/FileDataStore.hpp deleted 100644 → 0
View file @ 98e7aee4
#ifndef __HEPNOS_FILE_DATASTORE_H
#define __HEPNOS_FILE_DATASTORE_H
#include <string>
#include <iterator>
#include <boost/filesystem.hpp>
#include <hepnos/FileNamespace.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileDataStore {
public:
typedef FileNamespace namespace_type;
typedef FileRun run_type;
typedef FileSubRun subrun_type;
typedef FileEvent event_type;
FileDataStore(const std::string path)
: _path(path) {
if(_path.empty()) _path = "./";
if(_path.back() != '/') _path += std::string("/");
fs::create_directories(_path);
fs::create_directory(_path+std::string(".ref"));
}
namespace_type createNamespace(const std::string& name) {
std::string dir = _path + name;
if(fs::is_directory(dir))
return FileNamespace(name, dir);
fs::create_directory(dir);
return FileNamespace(name, dir);
}
namespace_type openNamespace(const std::string& name) {
std::string dir = _path + name;
if(fs::is_directory(dir))
return FileNamespace(name, dir);
else
return FileNamespace();
}
event_type getEventByID(std::uint64_t id) {
std::stringstream ss;
ss << _path << ".ref/" << std::setfill('0') << std::setw(20) << id;
std::string link = ss.str();
if(!fs::exists(link)) return FileEvent();
std::string p = fs::canonical(link).string();
return FileEvent(p);
}
private:
std::string _path;
};
}
#endif
include/hepnos/FileEvent.hpp deleted 100644 → 0
View file @ 98e7aee4
#ifndef __HEPNOS_FILE_EVENT_H
#define __HEPNOS_FILE_EVENT_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/FileObjectIterator.hpp>
#include <hepnos/ProductAccessor.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileSubRun;
class FileDataStore;
class FileEvent : public ProductAccessor<FileProductAccessorBackend> {
private:
friend class FileDataStore;
friend class FileSubRun;
friend class FileObjectIterator<FileEvent>;
void createRefAndSetID() {
std::size_t h = std::hash<std::string>()(_path);
std::string linkName;
std::stringstream ss;
do {
ss.str("");
ss << _path << "../../../../.ref/";
ss << std::setfill('0') << std::setw(20) << h;
h += 1;
} while(fs::exists(ss.str()));
_eventID = h-1;
fs::create_symlink(_path, ss.str());
}
FileEvent(std::uint64_t eventNumber, const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _eventNumber(eventNumber)
, _path(dir) {
createRefAndSetID();
}
FileEvent(const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _eventNumber(0)
, _path(dir) {
if(_path.back() != '/') _path += std::string("/");
std::size_t i,j;
j = _path.size()-1;
if(_path[j] == '/') j--;
i = j;
while(_path[i] != '/') i--;
i += 1;
while(_path[i] == '0') i++;
j += 1;
std::string eventDir(&dir[i], j-i);
if(eventDir.size() > 0)
_eventNumber = std::stoi(eventDir);
}
FileEvent()
: ProductAccessor<FileProductAccessorBackend>("")
, _eventNumber(std::numeric_limits<std::uint64_t>::max())
, _path("") {}
std::uint64_t _eventNumber;
std::string _path;
std::size_t _eventID;
public:
std::uint64_t getEventNumber() const {
return _eventNumber;
}
std::size_t getEventID() const {
return _eventID;
}
bool isValid() const {
return _eventNumber != std::numeric_limits<std::uint64_t>::max();
}
};
}
#endif
include/hepnos/FileNamespace.hpp deleted 100644 → 0
View file @ 98e7aee4
#ifndef __HEPNOS_FILE_NAMESPACE_H
#define __HEPNOS_FILE_NAMESPACE_H
#include <string>
#include <iomanip>
#include <boost/filesystem.hpp>
#include <hepnos/FileObjectIterator.hpp>
#include <hepnos/FileRun.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileDataStore;
class FileNamespace {
friend class FileDataStore;
private:
std::string _name;
std::string _path;
FileNamespace(const std::string& name, const std::string& dir)
: _name(name), _path(dir+std::string("/")) {}
FileNamespace() {}
public:
typedef FileObjectIterator<FileRun> iterator;
const std::string& getName() const {
return _name;
}
bool isValid() const {
return !_name.empty();
}
FileRun createRun() {
fs::directory_iterator begin(_path), end;
size_t runNumber = std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/";
std::string dir = ss.str();
fs::create_directory(dir);
return FileRun(runNumber, dir);
}
FileRun openRun(std::uint64_t runNumber) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/";
std::string dir = ss.str();
if(fs::is_directory(dir))
return FileRun(runNumber, dir);
else
return FileRun();
}
std::size_t numRuns() const {
fs::directory_iterator begin(_path), end;
return std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
}
iterator begin() {
return iterator(_path,0,numRuns());
}
iterator end() {
auto n = numRuns();
return iterator(_path,n,n);
}
};
}
#endif
include/hepnos/FileObjectIterator.hpp deleted 100644 → 0
View file @ 98e7aee4
#ifndef __HEPNOS_FILE_OBJECT_ITERATOR_H
#define __HEPNOS_FILE_OBJECT_ITERATOR_H
#include <boost/filesystem.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
template<typename T>
class FileObjectIterator {
private:
std::string _path;
std::uint64_t _numObjects;
std::uint64_t _currentIdx;
T _current;
public:
FileObjectIterator(const std::string& path, std::uint64_t idx, std::uint64_t nobj)
: _path(path), _numObjects(nobj), _currentIdx(idx) {
if(_path.back() != '/') _path += std::string("/");
if(_currentIdx != _numObjects) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
_current = T(_currentIdx, ss.str());
}
}
bool operator==(const FileObjectIterator& other) const {
return _path == other._path
&& _currentIdx == other._currentIdx
&& _numObjects == other._numObjects;
}
bool operator!=(const FileObjectIterator& other) const {
return !(*this == other);
}
const T& operator*() const {
return _current;
}
const T* operator->() const {
return &_current;
}
T* operator->() {
return &_current;
}
FileObjectIterator operator++() {
FileObjectIterator old = *this;
_currentIdx += 1;
if(_currentIdx != _numObjects) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
_current = T(_currentIdx, ss.str());
}
return old;
}
FileObjectIterator& operator++(int) {
_currentIdx += 1;
if(_currentIdx != _numObjects) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
_current = T(_currentIdx, ss.str());
}
return *this;
}
};
}
#endif
include/hepnos/FileProductAccessorBackend.hpp deleted 100644 → 0
View file @ 98e7aee4
#ifndef __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
#define __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
#include <vector>
#include <string>
#include <sstream>
#include <fstream>
#include <hepnos/InputTag.hpp>
namespace hepnos {
class FileProductAccessorBackend {
private:
std::string makeFileNameFor(const std::string& objName,
const InputTag& tag) {
std::string dot(".");
std::string fileName =
objName + dot + tag.moduleLabel + dot
+ tag.instanceName + dot + tag.processName;
if(_path.empty()) return fileName;
else if(_path.back() == '/') return _path + fileName;
else return _path + std::string("/") + fileName;
}
std::string _path;
public:
FileProductAccessorBackend(const std::string& path)
: _path(path) {}
void store(const std::string& objName,
const InputTag& tag,
const std::vector<char>& data) {
std::string fileName = makeFileNameFor(objName, tag);
std::ofstream outfile;
outfile.open(fileName, std::ofstream::out | std::ofstream::trunc);
outfile.write(data.data(), data.size());
outfile.close();
}
bool load(const std::string& objName,
const InputTag& tag,
std::vector<char>& data) {
std::string fileName = makeFileNameFor(objName, tag);
std::ifstream infile;
infile.open(fileName, std::ifstream::ate | std::ifstream::binary);
if(!infile.good()) return false;
std::size_t size = infile.tellg();
infile.seekg(0, infile.beg);
data.resize(size);
infile.read(data.data(), size);
infile.close();
return true;
}
~FileProductAccessorBackend() {}
};
}
#endif
include/hepnos/FileRun.hpp deleted 100644 → 0
View file @ 98e7aee4
#ifndef __HEPNOS_FILE_RUN_H
#define __HEPNOS_FILE_RUN_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/ProductAccessor.hpp>
#include <hepnos/FileSubRun.hpp>
#include <hepnos/FileObjectIterator.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileNamespace;
template<typename T> class FileObjectIterator;
class FileRun : public ProductAccessor<FileProductAccessorBackend> {
private:
friend class FileNamespace;
friend class FileObjectIterator<FileRun>;
FileRun(std::uint64_t runNumber, const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _runNumber(runNumber)
, _path(dir) {}
FileRun(const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _runNumber(0)
, _path(dir) {
std::size_t i,j;
j = dir.size()-1;
if(dir[j] == '/') j--;
i = j;
while(dir[i] != '/') i--;
i += 1;
while(dir[i] == '0') i++;
j += 1;
std::string runDir(&dir[i], j-i);
if(runDir.size() > 0)
_runNumber = std::stoi(runDir);
}
FileRun()
: ProductAccessor<FileProductAccessorBackend>("")
, _runNumber(std::numeric_limits<std::uint64_t>::max())
, _path("") {}
std::uint64_t _runNumber;
std::string _path;
public:
typedef FileObjectIterator<FileSubRun> iterator;
std::uint64_t getRunNumber() const {
return _runNumber;
}
bool isValid() const {
return _runNumber != std::numeric_limits<std::uint64_t>::max();
}
FileSubRun createSubRun() {
fs::directory_iterator begin(_path), end;
size_t subRunNumber = std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/";
std::string dir = ss.str();
fs::create_directory(dir);
return FileSubRun(subRunNumber, dir);
}
FileSubRun openSubRun(std::uint64_t subRunNumber) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/";
std::string dir = ss.str();
if(fs::is_directory(dir))
return FileSubRun(subRunNumber, dir);
else
return FileSubRun();
}
std::size_t numSubRuns() const {
fs::directory_iterator begin(_path), end;
return std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
}
iterator begin() {
return iterator(_path,0,numSubRuns());
}
iterator end() {
auto n = numSubRuns();
return iterator(_path,n,n);
}
};
}
#endif
include/hepnos/FileSubRun.hpp deleted 100644 → 0
View file @ 98e7aee4
#ifndef __HEPNOS_FILE_SUB_RUN_H
#define __HEPNOS_FILE_SUB_RUN_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/ProductAccessor.hpp>
#include <hepnos/FileEvent.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileRun;
class FileSubRun : public ProductAccessor<FileProductAccessorBackend> {
private:
friend class FileRun;
friend class FileObjectIterator<FileSubRun>;
FileSubRun(std::uint64_t subRunNumber, const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _subRunNumber(subRunNumber)