Commit 3542c123 authored by Matthieu Dorier's avatar Matthieu Dorier

first version, file-based

parent 381c3ba8
# general cmake flags:
# -DCMAKE_INSTALL_PREFIX=/usr/local -- the prefix for installing
# -DCMAKE_BUILD_TYPE=type -- type can be Debug, Release, ...
# -DCMAKE_PREFIX_PATH=/dir -- external packages
#
# note that CMAKE_PREFIX_PATH can be a list of directories:
# -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3'
#
cmake_minimum_required (VERSION 3.0)
project (hepnos CXX)
enable_testing ()
# add our cmake module directory to the path
set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
"${CMAKE_CURRENT_SOURCE_DIR}/cmake")
# link shared lib with full rpath
set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
# setup cache variables for ccmake
if (NOT CMAKE_BUILD_TYPE)
set (CMAKE_BUILD_TYPE Release
CACHE STRING "Choose the type of build." FORCE)
set_property (CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
"Debug" "Release" "RelWithDebInfo" "MinSizeRel")
endif ()
set (CMAKE_PREFIX_PATH "" CACHE STRING "External dependencies path")
set (BUILD_SHARED_LIBS "OFF" CACHE BOOL "Build a shared library")
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
# packages we depend on
include (xpkg-import)
find_package (mercury CONFIG REQUIRED)
find_package (Boost REQUIRED COMPONENTS serialization filesystem)
include_directories(${Boost_INCLUDE_DIRS})
xpkg_import_module (margo REQUIRED margo)
add_subdirectory (src)
add_subdirectory (test)
This diff is collapsed.
#ifndef __HEPNOS_FILE_DATASTORE_H
#define __HEPNOS_FILE_DATASTORE_H
#include <string>
#include <iterator>
#include <boost/filesystem.hpp>
#include <hepnos/FileNamespace.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileDataStore {
public:
typedef FileNamespace namespace_type;
typedef FileRun run_type;
typedef FileSubRun subrun_type;
typedef FileEvent event_type;
FileDataStore(const std::string path)
: _path(path) {
if(_path.empty()) _path = "./";
if(_path.back() != '/') _path += std::string("/");
fs::create_directories(_path);
fs::create_directory(_path+std::string(".ref"));
}
namespace_type createNamespace(const std::string& name) {
std::string dir = _path + name;
if(fs::is_directory(dir))
return FileNamespace(name, dir);
fs::create_directory(dir);
return FileNamespace(name, dir);
}
namespace_type openNamespace(const std::string& name) {
std::string dir = _path + name;
if(fs::is_directory(dir))
return FileNamespace(name, dir);
else
return FileNamespace();
}
private:
std::string _path;
};
}
#endif
#ifndef __HEPNOS_FILE_EVENT_H
#define __HEPNOS_FILE_EVENT_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/FileObjectIterator.hpp>
#include <hepnos/ProductAccessor.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileSubRun;
class FileEvent : public ProductAccessor<FileProductAccessorBackend> {
private:
friend class FileSubRun;
friend class FileObjectIterator<FileEvent>;
void createRefAndSetID() {
std::size_t h = std::hash<std::string>()(_path);
std::string linkName;
std::stringstream ss;
do {
ss.str("");
ss << _path << "../../../../.ref/";
ss << std::setfill('0') << std::setw(20) << h;
h += 1;
} while(fs::exists(ss.str()));
_eventID = h-1;
fs::create_symlink(_path, ss.str());
}
FileEvent(std::uint64_t eventNumber, const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _eventNumber(eventNumber)
, _path(dir) {
createRefAndSetID();
}
FileEvent()
: ProductAccessor<FileProductAccessorBackend>("")
, _eventNumber(std::numeric_limits<std::uint64_t>::max())
, _path("") {}
std::uint64_t _eventNumber;
std::string _path;
std::size_t _eventID;
public:
std::uint64_t getEventNumber() const {
return _eventNumber;
}
std::size_t getEventID() const {
return _eventID;
}
bool isValid() const {
return _eventNumber != std::numeric_limits<std::uint64_t>::max();
}
};
}
#endif
#ifndef __HEPNOS_FILE_NAMESPACE_H
#define __HEPNOS_FILE_NAMESPACE_H
#include <string>
#include <iomanip>
#include <boost/filesystem.hpp>
#include <hepnos/FileObjectIterator.hpp>
#include <hepnos/FileRun.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileDataStore;
class FileNamespace {
friend class FileDataStore;
private:
std::string _name;
std::string _path;
FileNamespace(const std::string& name, const std::string& dir)
: _name(name), _path(dir+std::string("/")) {}
FileNamespace() {}
public:
typedef FileObjectIterator<FileRun> iterator;
const std::string& getName() const {
return _name;
}
bool isValid() const {
return !_name.empty();
}
FileRun createRun() {
fs::directory_iterator begin(_path), end;
size_t runNumber = std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/";
std::string dir = ss.str();
fs::create_directory(dir);
return FileRun(runNumber, dir);
}
FileRun openRun(std::uint64_t runNumber) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/";
std::string dir = ss.str();
if(fs::is_directory(dir))
return FileRun(runNumber, dir);
else
return FileRun();
}
std::size_t numRuns() const {
fs::directory_iterator begin(_path), end;
return std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
}
iterator begin() {
return iterator(_path,0,numRuns());
}
iterator end() {
auto n = numRuns();
return iterator(_path,n,n);
}
};
}
#endif
#ifndef __HEPNOS_FILE_OBJECT_ITERATOR_H
#define __HEPNOS_FILE_OBJECT_ITERATOR_H
#include <boost/filesystem.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
template<typename T>
class FileObjectIterator {
private:
std::string _path;
std::uint64_t _numObjects;
std::uint64_t _currentIdx;
T _current;
public:
FileObjectIterator(const std::string& path, std::uint64_t idx, std::uint64_t nobj)
: _path(path), _numObjects(nobj), _currentIdx(idx) {
if(_path.back() != '/') _path += std::string("/");
if(_currentIdx != _numObjects) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
_current = T(_currentIdx, ss.str());
}
}
bool operator==(const FileObjectIterator& other) const {
return _path == other._path
&& _currentIdx == other._currentIdx
&& _numObjects == other._numObjects;
}
bool operator!=(const FileObjectIterator& other) const {
return !(*this == other);
}
const T& operator*() const {
return _current;
}
const T* operator->() const {
return &_current;
}
T* operator->() {
return &_current;
}
FileObjectIterator operator++() {
FileObjectIterator old = *this;
_currentIdx += 1;
if(_currentIdx != _numObjects) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
_current = T(_currentIdx, ss.str());
}
return old;
}
FileObjectIterator& operator++(int) {
_currentIdx += 1;
if(_currentIdx != _numObjects) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
_current = T(_currentIdx, ss.str());
}
return *this;
}
};
}
#endif
#ifndef __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
#define __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
#include <vector>
#include <string>
#include <sstream>
#include <fstream>
#include <hepnos/InputTag.hpp>
namespace hepnos {
class FileProductAccessorBackend {
private:
std::string makeFileNameFor(const std::string& objName,
const InputTag& tag) {
std::string dot(".");
std::string fileName =
objName + dot + tag.moduleLabel + dot
+ tag.instanceName + dot + tag.processName;
if(_path.empty()) return fileName;
else if(_path.back() == '/') return _path + fileName;
else return _path + std::string("/") + fileName;
}
std::string _path;
public:
FileProductAccessorBackend(const std::string& path)
: _path(path) {}
void store(const std::string& objName,
const InputTag& tag,
const std::vector<char>& data) {
std::string fileName = makeFileNameFor(objName, tag);
std::ofstream outfile;
outfile.open(fileName, std::ofstream::out | std::ofstream::trunc);
outfile.write(data.data(), data.size());
outfile.close();
}
bool load(const std::string& objName,
const InputTag& tag,
std::vector<char>& data) {
std::string fileName = makeFileNameFor(objName, tag);
std::ifstream infile;
infile.open(fileName, std::ifstream::ate | std::ifstream::binary);
if(!infile.good()) return false;
std::size_t size = infile.tellg();
infile.seekg(0, infile.beg);
data.resize(size);
infile.read(data.data(), size);
infile.close();
return true;
}
~FileProductAccessorBackend() {}
};
}
#endif
#ifndef __HEPNOS_FILE_RUN_H
#define __HEPNOS_FILE_RUN_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/ProductAccessor.hpp>
#include <hepnos/FileSubRun.hpp>
#include <hepnos/FileObjectIterator.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileNamespace;
template<typename T> class FileObjectIterator;
class FileRun : public ProductAccessor<FileProductAccessorBackend> {
private:
friend class FileNamespace;
friend class FileObjectIterator<FileRun>;
FileRun(std::uint64_t runNumber, const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _runNumber(runNumber)
, _path(dir) {}
FileRun(const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _runNumber(0)
, _path(dir) {
std::size_t i,j;
j = dir.size()-1;
if(dir[j] == '/') j--;
i = j;
while(dir[i] != '/') i--;
i += 1;
while(dir[i] == '0') i++;
j += 1;
std::string runDir(&dir[i], j-i);
if(runDir.size() > 0)
_runNumber = std::stoi(runDir);
}
FileRun()
: ProductAccessor<FileProductAccessorBackend>("")
, _runNumber(std::numeric_limits<std::uint64_t>::max())
, _path("") {}
std::uint64_t _runNumber;
std::string _path;
public:
typedef FileObjectIterator<FileSubRun> iterator;
std::uint64_t getRunNumber() const {
return _runNumber;
}
bool isValid() const {
return _runNumber != std::numeric_limits<std::uint64_t>::max();
}
FileSubRun createSubRun() {
fs::directory_iterator begin(_path), end;
size_t subRunNumber = std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/";
std::string dir = ss.str();
fs::create_directory(dir);
return FileSubRun(subRunNumber, dir);
}
FileSubRun openSubRun(std::uint64_t subRunNumber) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/";
std::string dir = ss.str();
if(fs::is_directory(dir))
return FileSubRun(subRunNumber, dir);
else
return FileSubRun();
}
std::size_t numSubRuns() const {
fs::directory_iterator begin(_path), end;
return std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
}
iterator begin() {
return iterator(_path,0,numSubRuns());
}
iterator end() {
auto n = numSubRuns();
return iterator(_path,n,n);
}
};
}
#endif
#ifndef __HEPNOS_FILE_SUB_RUN_H
#define __HEPNOS_FILE_SUB_RUN_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/ProductAccessor.hpp>
#include <hepnos/FileEvent.hpp>
namespace hepnos {
namespace fs = boost::filesystem;
class FileRun;
class FileSubRun : public ProductAccessor<FileProductAccessorBackend> {
private:
friend class FileRun;
friend class FileObjectIterator<FileSubRun>;
FileSubRun(std::uint64_t subRunNumber, const std::string& dir)
: ProductAccessor<FileProductAccessorBackend>(dir)
, _subRunNumber(subRunNumber)
, _path(dir) {}
FileSubRun()
: ProductAccessor<FileProductAccessorBackend>("")
, _subRunNumber(std::numeric_limits<std::uint64_t>::max())
, _path("") {}
std::uint64_t _subRunNumber;
std::string _path;
public:
typedef FileObjectIterator<FileEvent> iterator;
std::uint64_t getSubRunNumber() const {
return _subRunNumber;
}
bool isValid() const {
return _subRunNumber != std::numeric_limits<std::uint64_t>::max();
}
FileEvent createEvent() {
fs::directory_iterator begin(_path), end;
size_t eventNumber = std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << eventNumber << "/";
std::string dir = ss.str();
fs::create_directory(dir);
return FileEvent(eventNumber, dir);
}
FileEvent openEvent(std::uint64_t eventNumber) {
std::stringstream ss;
ss << _path << std::setfill('0') << std::setw(12) << eventNumber << "/";
std::string dir = ss.str();
if(fs::is_directory(dir))
return FileEvent(eventNumber, dir);
else
return FileEvent();
}
std::size_t numEvents() const {
fs::directory_iterator begin(_path), end;
return std::count_if(begin, end,
[](const fs::directory_entry& d) {
return fs::is_directory(d.path());
});
}
iterator begin() {
return iterator(_path, 0, numEvents());
}
iterator end() {
auto n = numEvents();
return iterator(_path, n, n);
}
};
}
#endif
#ifndef __HEPNOS_INPUT_TAG_H
#define __HEPNOS_INPUT_TAG_H
#include <string>
namespace hepnos {
struct InputTag {
std::string moduleLabel;
std::string instanceName;
std::string processName;
InputTag(const std::string& ml,
const std::string& in,
const std::string& pn)
: moduleLabel(ml),
instanceName(in),
processName(pn) {}
};
}
#endif
#ifndef __HEPNOS_PRODUCT_ACCESSOR_H
#define __HEPNOS_PRODUCT_ACCESSOR_H
#include <fstream>
#include <string>
#include <typeinfo>
#include <cxxabi.h>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <hepnos/InputTag.hpp>
namespace hepnos {
/**
* This function takes an object name created by typeid(obj).name()
* and returns a "friendly" (i.e. compiler-independent) version of
* this name.
*/
static std::string demangle(const char* name) {
int status = -4;
char* res = abi::__cxa_demangle(name, NULL, NULL, &status);
const char* const demangled_name = (status==0)?res:name;
std::string ret_val(demangled_name);
free(res);
return ret_val;
}
template<typename BackendProductAccessor>
class ProductAccessor {
private:
BackendProductAccessor _backend;
public:
template<typename ... Args>
ProductAccessor(Args&& ... args)
: _backend(std::forward<Args>(args)...) {}
template<typename T>
void store(const InputTag& tag, const T& obj) {
std::string objType = demangle(typeid(obj).name());
std::stringstream ss;
boost::archive::binary_oarchive oa(ss);
oa << obj;
std::string serialized = ss.str();
std::vector<char> buffer(serialized.begin(), serialized.end());
_backend.store(objType, tag, buffer);
}
template<typename T>
bool load(const InputTag& tag, T& obj) {
std::string objType = demangle(typeid(obj).name());
std::vector<char> buffer;
if(!_backend.load(objType, tag, buffer)) return false;
std::string serialized(buffer.begin(), buffer.end());
std::stringstream ss(serialized);
boost::archive::binary_iarchive ia(ss);
ia >> obj;
return true;
}
};
}
#endif
# list of source files
set(hepnos-server-src hepnos-server.c)
# load package helper for generating cmake CONFIG packages
include (CMakePackageConfigHelpers)
# where to install files for "find_package"
set (hepnos-pkg "share/cmake/hepnos")
#
# library version set here (e.g. for shared libs).
#
set (HEPNOS_VERSION_MAJOR 1)
set (HEPNOS_VERSION_MINOR 0)
set (HEPNOS_VERSION_PATCH 0)
set (hepnos-vers "${HEPNOS_VERSION_MAJOR}.${HEPNOS_VERSION_MINOR}")
set (HEPNOS_VERSION "${hepnos-vers}.${HEPNOS_VERSION_PATCH}")
#add_library(hepnos ${hepnos-src})
#target_link_libraries (hepnos mercury margo)
#target_include_directories (hepnos PUBLIC $<INSTALL_INTERFACE:include>)
# local include's BEFORE, in case old incompatable .h files in prefix/include
#target_include_directories (hepnos BEFORE PUBLIC
# $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>)
# for shared libs, establish the lib version
#set_target_properties (hepnos
# PROPERTIES VERSION ${HEPNOS_VERSION}
# SOVERSION ${HEPNOS_VERSION_MAJOR})