first version, file-based

CMakeLists.txt

+#  general cmake flags:
+#    -DCMAKE_INSTALL_PREFIX=/usr/local     -- the prefix for installing
+#    -DCMAKE_BUILD_TYPE=type               -- type can be Debug, Release, ...
+#    -DCMAKE_PREFIX_PATH=/dir              -- external packages
+#
+#     note that CMAKE_PREFIX_PATH can be a list of directories:
+#      -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3'
+#
+
+cmake_minimum_required (VERSION 3.0)
+project (hepnos CXX)
+enable_testing ()
+
+# add our cmake module directory to the path
+set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
+     "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+# link shared lib with full rpath
+set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
+# setup cache variables for ccmake
+if (NOT CMAKE_BUILD_TYPE)
+    set (CMAKE_BUILD_TYPE Release
+         CACHE STRING "Choose the type of build." FORCE)
+    set_property (CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS 
+                  "Debug" "Release" "RelWithDebInfo" "MinSizeRel")
+endif ()
+set (CMAKE_PREFIX_PATH "" CACHE STRING "External dependencies path")
+set (BUILD_SHARED_LIBS "OFF" CACHE BOOL "Build a shared library")
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
+
+# packages we depend on
+include (xpkg-import)
+find_package (mercury CONFIG REQUIRED)
+find_package (Boost REQUIRED COMPONENTS serialization filesystem)
+include_directories(${Boost_INCLUDE_DIRS})
+xpkg_import_module (margo REQUIRED margo)
+
+add_subdirectory (src)
+add_subdirectory (test)

include/hepnos/FileDataStore.hpp

+#ifndef __HEPNOS_FILE_DATASTORE_H
+#define __HEPNOS_FILE_DATASTORE_H
+
+#include <string>
+#include <iterator>
+#include <boost/filesystem.hpp>
+#include <hepnos/FileNamespace.hpp>
+
+namespace hepnos {
+
+namespace fs = boost::filesystem;
+
+class FileDataStore {
+
+public:
+
+    typedef FileNamespace namespace_type;
+    typedef FileRun       run_type;
+    typedef FileSubRun    subrun_type;
+    typedef FileEvent     event_type;
+
+    FileDataStore(const std::string path)
+    : _path(path) {
+        if(_path.empty()) _path = "./";
+        if(_path.back() != '/') _path += std::string("/");
+        fs::create_directories(_path);
+        fs::create_directory(_path+std::string(".ref"));
+    }
+
+    namespace_type createNamespace(const std::string& name) {
+        std::string dir = _path + name;
+        if(fs::is_directory(dir)) 
+            return FileNamespace(name, dir);
+        fs::create_directory(dir);
+        return FileNamespace(name, dir);
+    }
+
+    namespace_type openNamespace(const std::string& name) {
+        std::string dir = _path + name;
+        if(fs::is_directory(dir))
+            return FileNamespace(name, dir);
+        else
+            return FileNamespace();
+    }
+
+private:
+
+    std::string _path;
+};
+
+}
+
+#endif

include/hepnos/FileEvent.hpp

+#ifndef __HEPNOS_FILE_EVENT_H
+#define __HEPNOS_FILE_EVENT_H
+
+#include <string>
+#include <limits>
+#include <boost/filesystem.hpp>
+#include <hepnos/FileProductAccessorBackend.hpp>
+#include <hepnos/FileObjectIterator.hpp>
+#include <hepnos/ProductAccessor.hpp>
+
+namespace hepnos {
+
+namespace fs = boost::filesystem;
+
+class FileSubRun;
+
+class FileEvent : public ProductAccessor<FileProductAccessorBackend> {
+
+    private:
+
+        friend class FileSubRun;
+        friend class FileObjectIterator<FileEvent>;
+
+        void createRefAndSetID() {
+            std::size_t h = std::hash<std::string>()(_path);
+            std::string linkName;
+            std::stringstream ss;
+            do {
+                ss.str("");
+                ss << _path << "../../../../.ref/";
+                ss << std::setfill('0') << std::setw(20) << h;
+                h += 1;
+            } while(fs::exists(ss.str()));
+            _eventID = h-1;
+            fs::create_symlink(_path, ss.str());
+        }
+
+        FileEvent(std::uint64_t eventNumber, const std::string& dir) 
+        : ProductAccessor<FileProductAccessorBackend>(dir)
+        , _eventNumber(eventNumber)
+        , _path(dir) {
+            createRefAndSetID();
+        }
+
+        FileEvent()
+        : ProductAccessor<FileProductAccessorBackend>("")
+        , _eventNumber(std::numeric_limits<std::uint64_t>::max())
+        , _path("") {}
+
+        std::uint64_t _eventNumber;
+        std::string   _path;
+        std::size_t   _eventID;
+
+    public:
+
+        std::uint64_t getEventNumber() const {
+            return _eventNumber;
+        }
+
+        std::size_t getEventID() const {
+            return _eventID;
+        }
+
+        bool isValid() const {
+            return _eventNumber != std::numeric_limits<std::uint64_t>::max();
+        }
+};
+
+}
+#endif

include/hepnos/FileNamespace.hpp

+#ifndef __HEPNOS_FILE_NAMESPACE_H
+#define __HEPNOS_FILE_NAMESPACE_H
+
+#include <string>
+#include <iomanip>
+#include <boost/filesystem.hpp>
+#include <hepnos/FileObjectIterator.hpp>
+#include <hepnos/FileRun.hpp>
+
+namespace hepnos {
+
+namespace fs = boost::filesystem;
+
+class FileDataStore;
+
+class FileNamespace {
+
+    friend class FileDataStore;
+
+    private:
+
+        std::string _name;
+        std::string _path;
+
+        FileNamespace(const std::string& name, const std::string& dir)
+        : _name(name), _path(dir+std::string("/")) {}
+
+        FileNamespace() {}
+
+    public:
+
+        typedef FileObjectIterator<FileRun> iterator;
+
+        const std::string& getName() const {
+            return _name;
+        }
+
+        bool isValid() const {
+            return !_name.empty();
+        }
+
+        FileRun createRun() {
+            fs::directory_iterator begin(_path), end;
+            size_t runNumber = std::count_if(begin, end,
+                    [](const fs::directory_entry& d) {
+                        return fs::is_directory(d.path());
+                    });
+            std::stringstream ss;
+            ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/";
+            std::string dir = ss.str();
+            fs::create_directory(dir);
+            return FileRun(runNumber, dir);
+        }
+
+        FileRun openRun(std::uint64_t runNumber) {
+            std::stringstream ss;
+            ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/";
+            std::string dir = ss.str();
+            if(fs::is_directory(dir))
+                return FileRun(runNumber, dir);
+            else
+                return FileRun();
+        }
+
+        std::size_t numRuns() const {
+            fs::directory_iterator begin(_path), end;
+            return std::count_if(begin, end,
+                    [](const fs::directory_entry& d) {
+                        return fs::is_directory(d.path());
+                    });
+        }
+
+        iterator begin() {
+            return iterator(_path,0,numRuns());
+        }
+
+        iterator end() {
+            auto n = numRuns();
+            return iterator(_path,n,n);
+        }
+};
+
+}
+#endif

include/hepnos/FileObjectIterator.hpp

+#ifndef __HEPNOS_FILE_OBJECT_ITERATOR_H
+#define __HEPNOS_FILE_OBJECT_ITERATOR_H
+
+#include <boost/filesystem.hpp>
+
+namespace hepnos {
+
+namespace fs = boost::filesystem;
+
+template<typename T>
+class FileObjectIterator {
+
+    private:
+
+        std::string   _path;
+        std::uint64_t _numObjects;
+        std::uint64_t _currentIdx;
+        T             _current;
+
+    public:
+
+        FileObjectIterator(const std::string& path, std::uint64_t idx, std::uint64_t nobj)
+        : _path(path), _numObjects(nobj), _currentIdx(idx) {
+            if(_path.back() != '/') _path += std::string("/");
+            if(_currentIdx != _numObjects) {
+                std::stringstream ss;
+                ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
+                _current = T(_currentIdx, ss.str());
+            }
+        }
+
+        bool operator==(const FileObjectIterator& other) const {
+            return _path == other._path
+                && _currentIdx == other._currentIdx
+                && _numObjects == other._numObjects;
+        }
+
+        bool operator!=(const FileObjectIterator& other) const {
+            return !(*this == other);
+        }
+
+        const T& operator*() const {
+            return _current;
+        }
+
+        const T* operator->() const {
+            return &_current;
+        }
+
+        T* operator->() {
+            return &_current;
+        }
+
+        FileObjectIterator operator++() {
+            FileObjectIterator old = *this;
+            _currentIdx += 1;
+            if(_currentIdx != _numObjects) {
+                std::stringstream ss;
+                ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/";
+                _current = T(_currentIdx, ss.str());
+            }
+            return old;
+        }
+
+        FileObjectIterator& operator++(int) {
+            _currentIdx += 1;
+            if(_currentIdx != _numObjects) {
+                std::stringstream ss;
+                ss << _path << std::setfill('0') << std::setw(12) <<  _currentIdx << "/";
+                _current = T(_currentIdx, ss.str());
+            }
+            return *this;
+        }
+};
+
+}
+
+#endif

include/hepnos/FileProductAccessorBackend.hpp

+#ifndef __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
+#define __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
+
+#include <vector>
+#include <string>
+#include <sstream>
+#include <fstream>
+#include <hepnos/InputTag.hpp>
+
+namespace hepnos {
+
+class FileProductAccessorBackend {
+    
+    private:
+
+        std::string makeFileNameFor(const std::string& objName,
+                                    const InputTag& tag) {
+            std::string dot(".");
+            std::string fileName = 
+                objName + dot + tag.moduleLabel + dot
+                + tag.instanceName + dot + tag.processName;
+            if(_path.empty()) return fileName;
+            else if(_path.back() == '/') return _path + fileName;
+            else return _path + std::string("/") + fileName;
+        }
+
+        std::string _path;
+
+    public:
+
+        FileProductAccessorBackend(const std::string& path)
+        : _path(path) {}
+
+        void store(const std::string& objName,
+                   const InputTag& tag,
+                   const std::vector<char>& data) {
+            std::string fileName = makeFileNameFor(objName, tag);
+            std::ofstream outfile;
+            outfile.open(fileName, std::ofstream::out | std::ofstream::trunc);
+            outfile.write(data.data(), data.size());
+            outfile.close();
+        }
+
+        bool load(const std::string& objName,
+                  const InputTag& tag,
+                  std::vector<char>& data) {
+            std::string fileName = makeFileNameFor(objName, tag);
+            std::ifstream infile;
+            infile.open(fileName, std::ifstream::ate | std::ifstream::binary);
+            if(!infile.good()) return false;
+            std::size_t size = infile.tellg();
+            infile.seekg(0, infile.beg);
+            data.resize(size);
+            infile.read(data.data(), size);
+            infile.close();
+            return true;
+        }
+
+        ~FileProductAccessorBackend() {}
+
+};
+
+}
+#endif

include/hepnos/FileRun.hpp

+#ifndef __HEPNOS_FILE_RUN_H
+#define __HEPNOS_FILE_RUN_H
+
+#include <string>
+#include <limits>
+#include <boost/filesystem.hpp>
+#include <hepnos/FileProductAccessorBackend.hpp>
+#include <hepnos/ProductAccessor.hpp>
+#include <hepnos/FileSubRun.hpp>
+#include <hepnos/FileObjectIterator.hpp>
+
+namespace hepnos {
+
+namespace fs = boost::filesystem;
+
+class FileNamespace;
+template<typename T> class FileObjectIterator;
+
+class FileRun : public ProductAccessor<FileProductAccessorBackend> {
+
+    private:
+
+        friend class FileNamespace;
+        friend class FileObjectIterator<FileRun>;
+
+        FileRun(std::uint64_t runNumber, const std::string& dir) 
+        : ProductAccessor<FileProductAccessorBackend>(dir)
+        , _runNumber(runNumber)
+        , _path(dir) {}
+
+        FileRun(const std::string& dir)
+        : ProductAccessor<FileProductAccessorBackend>(dir)
+        , _runNumber(0)
+        , _path(dir) {
+            std::size_t i,j;
+            j = dir.size()-1;
+            if(dir[j] == '/') j--;
+            i = j;
+            while(dir[i] != '/') i--;
+            i += 1;
+            while(dir[i] == '0') i++;
+            j += 1;
+            std::string runDir(&dir[i], j-i);
+            if(runDir.size() > 0)
+                _runNumber = std::stoi(runDir);
+        }
+
+        FileRun() 
+        : ProductAccessor<FileProductAccessorBackend>("")
+        , _runNumber(std::numeric_limits<std::uint64_t>::max())
+        , _path("") {}
+
+        std::uint64_t _runNumber;
+        std::string _path;
+
+    public:
+
+        typedef FileObjectIterator<FileSubRun> iterator;
+
+        std::uint64_t getRunNumber() const {
+            return _runNumber;
+        }
+
+        bool isValid() const {
+            return _runNumber != std::numeric_limits<std::uint64_t>::max();
+        }
+
+        FileSubRun createSubRun() {
+            fs::directory_iterator begin(_path), end;
+            size_t subRunNumber = std::count_if(begin, end,
+                    [](const fs::directory_entry& d) {
+                    return fs::is_directory(d.path());
+                    });
+            std::stringstream ss;
+            ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/";
+            std::string dir = ss.str();
+            fs::create_directory(dir);
+            return FileSubRun(subRunNumber, dir);
+        }
+
+        FileSubRun openSubRun(std::uint64_t subRunNumber) {
+            std::stringstream ss;
+            ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/";
+            std::string dir = ss.str();
+            if(fs::is_directory(dir))
+                return FileSubRun(subRunNumber, dir);
+            else
+                return FileSubRun();
+        }
+
+        std::size_t numSubRuns() const {
+            fs::directory_iterator begin(_path), end;
+            return std::count_if(begin, end,
+                    [](const fs::directory_entry& d) {
+                    return fs::is_directory(d.path());
+                    });
+        }
+
+        iterator begin() {
+            return iterator(_path,0,numSubRuns());
+        }
+
+        iterator end() {
+            auto n = numSubRuns();
+            return iterator(_path,n,n);
+        }
+};
+
+}
+#endif

include/hepnos/FileSubRun.hpp

+#ifndef __HEPNOS_FILE_SUB_RUN_H
+#define __HEPNOS_FILE_SUB_RUN_H
+
+#include <string>
+#include <limits>
+#include <boost/filesystem.hpp>
+#include <hepnos/FileProductAccessorBackend.hpp>
+#include <hepnos/ProductAccessor.hpp>
+#include <hepnos/FileEvent.hpp>
+
+namespace hepnos {
+
+namespace fs = boost::filesystem;
+
+class FileRun;
+
+class FileSubRun : public ProductAccessor<FileProductAccessorBackend> {
+
+    private:
+
+        friend class FileRun;
+        friend class FileObjectIterator<FileSubRun>;
+
+        FileSubRun(std::uint64_t subRunNumber, const std::string& dir) 
+        : ProductAccessor<FileProductAccessorBackend>(dir)
+        , _subRunNumber(subRunNumber)
+        , _path(dir) {}
+
+        FileSubRun()
+        : ProductAccessor<FileProductAccessorBackend>("")
+        , _subRunNumber(std::numeric_limits<std::uint64_t>::max())
+        , _path("") {}
+
+        std::uint64_t _subRunNumber;
+        std::string   _path;
+
+    public:
+
+        typedef FileObjectIterator<FileEvent> iterator;
+
+        std::uint64_t getSubRunNumber() const {
+            return _subRunNumber;
+        }
+
+        bool isValid() const {
+            return _subRunNumber != std::numeric_limits<std::uint64_t>::max();
+        }
+
+        FileEvent createEvent() {
+            fs::directory_iterator begin(_path), end;
+            size_t eventNumber = std::count_if(begin, end,
+                    [](const fs::directory_entry& d) {
+                    return fs::is_directory(d.path());
+                    });
+            std::stringstream ss;
+            ss << _path << std::setfill('0') << std::setw(12) << eventNumber << "/";
+            std::string dir = ss.str();
+            fs::create_directory(dir);
+            return FileEvent(eventNumber, dir);
+        }
+
+        FileEvent openEvent(std::uint64_t eventNumber) {
+            std::stringstream ss;
+            ss << _path << std::setfill('0') << std::setw(12) << eventNumber << "/";
+            std::string dir = ss.str();
+            if(fs::is_directory(dir))
+                return FileEvent(eventNumber, dir);
+            else
+                return FileEvent();
+        }
+
+        std::size_t numEvents() const {
+            fs::directory_iterator begin(_path), end;
+            return std::count_if(begin, end,
+                    [](const fs::directory_entry& d) {
+                    return fs::is_directory(d.path());
+                    });
+        }
+
+        iterator begin() {
+            return iterator(_path, 0, numEvents());
+        }
+
+        iterator end() {
+            auto n = numEvents();
+            return iterator(_path, n, n);
+        }
+};
+
+}
+#endif

include/hepnos/InputTag.hpp

+#ifndef __HEPNOS_INPUT_TAG_H
+#define __HEPNOS_INPUT_TAG_H
+
+#include <string>
+
+namespace hepnos {
+
+struct InputTag {
+
+    std::string moduleLabel;
+    std::string instanceName;
+    std::string processName;
+
+    InputTag(const std::string& ml,
+             const std::string& in,
+             const std::string& pn)
+        : moduleLabel(ml),
+          instanceName(in),
+          processName(pn) {}
+};
+
+}
+
+#endif

include/hepnos/ProductAccessor.hpp

+#ifndef __HEPNOS_PRODUCT_ACCESSOR_H
+#define __HEPNOS_PRODUCT_ACCESSOR_H
+
+#include <fstream>
+#include <string>
+#include <typeinfo>
+#include <cxxabi.h>
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/archive/binary_iarchive.hpp>
+#include <boost/archive/text_oarchive.hpp>
+#include <boost/archive/text_iarchive.hpp>
+#include <hepnos/InputTag.hpp>
+
+namespace hepnos {
+
+/**
+ * This function takes an object name created by typeid(obj).name()
+ * and returns a "friendly" (i.e. compiler-independent) version of
+ * this name.
+ */
+static std::string demangle(const char* name) {
+    int status = -4;
+    char* res = abi::__cxa_demangle(name, NULL, NULL, &status);
+    const char* const demangled_name = (status==0)?res:name;
+    std::string ret_val(demangled_name);
+    free(res);
+    return ret_val;
+}
+
+template<typename BackendProductAccessor>
+class ProductAccessor {
+
+    private:
+
+        BackendProductAccessor _backend;
+
+    public:
+
+        template<typename ... Args>
+        ProductAccessor(Args&& ... args)
+        : _backend(std::forward<Args>(args)...) {}
+
+        template<typename T>
+        void store(const InputTag& tag, const T& obj) {
+            std::string objType = demangle(typeid(obj).name());
+            std::stringstream ss;
+            boost::archive::binary_oarchive oa(ss);
+            oa << obj;
+            std::string serialized = ss.str();
+            std::vector<char> buffer(serialized.begin(), serialized.end());
+            _backend.store(objType, tag, buffer);
+        }
+
+        template<typename T>
+        bool load(const InputTag& tag, T& obj) {
+            std::string objType = demangle(typeid(obj).name());
+            std::vector<char> buffer;
+            if(!_backend.load(objType, tag, buffer)) return false;
+            std::string serialized(buffer.begin(), buffer.end());
+            std::stringstream ss(serialized);
+            boost::archive::binary_iarchive ia(ss);
+            ia >> obj;
+            return true;
+        }
+};
+
+}
+#endif

src/CMakeLists.txt

+# list of source files
+set(hepnos-server-src hepnos-server.c)
+
+# load package helper for generating cmake CONFIG packages
+include (CMakePackageConfigHelpers)
+
+# where to install files for "find_package"
+set (hepnos-pkg "share/cmake/hepnos")
+
+#
+# library version set here (e.g. for shared libs).
+#
+set (HEPNOS_VERSION_MAJOR 1)
+set (HEPNOS_VERSION_MINOR 0)
+set (HEPNOS_VERSION_PATCH 0)