From 651208a78008bbe074dee547412ffedbb1478972 Mon Sep 17 00:00:00 2001 From: Matthieu Dorier Date: Tue, 10 Apr 2018 17:32:37 +0200 Subject: [PATCH] started implementation of real mochi-based storage system --- CMakeLists.txt | 9 +- include/hepnos/DataSet.hpp | 23 ++ include/hepnos/DataStore.hpp | 119 ++++++ include/hepnos/Exception.hpp | 25 ++ include/hepnos/FileDataStore.hpp | 62 --- include/hepnos/FileEvent.hpp | 90 ---- include/hepnos/FileNamespace.hpp | 84 ---- include/hepnos/FileObjectIterator.hpp | 78 ---- include/hepnos/FileProductAccessorBackend.hpp | 64 --- include/hepnos/FileRun.hpp | 110 ----- include/hepnos/FileSubRun.hpp | 91 ---- include/hepnos/InputTag.hpp | 24 -- include/hepnos/ProductAccessor.hpp | 68 --- src/CMakeLists.txt | 8 +- src/DataSet.cpp | 10 + src/DataStore.cpp | 388 ++++++++++++++++++ src/private/KeyTypes.hpp | 47 +++ test/CMakeLists.txt | 2 + test/config.yml | 6 + test/example.cpp | 11 + 20 files changed, 643 insertions(+), 676 deletions(-) create mode 100644 include/hepnos/DataSet.hpp create mode 100644 include/hepnos/DataStore.hpp create mode 100644 include/hepnos/Exception.hpp delete mode 100644 include/hepnos/FileDataStore.hpp delete mode 100644 include/hepnos/FileEvent.hpp delete mode 100644 include/hepnos/FileNamespace.hpp delete mode 100644 include/hepnos/FileObjectIterator.hpp delete mode 100644 include/hepnos/FileProductAccessorBackend.hpp delete mode 100644 include/hepnos/FileRun.hpp delete mode 100644 include/hepnos/FileSubRun.hpp delete mode 100644 include/hepnos/InputTag.hpp delete mode 100644 include/hepnos/ProductAccessor.hpp create mode 100644 src/DataSet.cpp create mode 100644 src/DataStore.cpp create mode 100644 src/private/KeyTypes.hpp create mode 100644 test/CMakeLists.txt create mode 100644 test/config.yml create mode 100644 test/example.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 10a5f68..584e8df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,9 +34,16 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) # packages we depend on include (xpkg-import) find_package (mercury CONFIG REQUIRED) -find_package (Boost REQUIRED COMPONENTS serialization filesystem) +find_package (Boost REQUIRED COMPONENTS serialization system filesystem) include_directories(${Boost_INCLUDE_DIRS}) xpkg_import_module (margo REQUIRED margo) +xpkg_import_module (sdskv-client REQUIRED sdskv-client) +xpkg_import_module (bake-client REQUIRED bake-client) +xpkg_import_module (sdskv-server REQUIRED sdskv-server) +xpkg_import_module (bake-server REQUIRED bake-server) +xpkg_import_module (ch-placement REQUIRED ch-placement) +find_package (yaml-cpp REQUIRED) add_subdirectory (src) add_subdirectory (test) +#add_subdirectory (bin) diff --git a/include/hepnos/DataSet.hpp b/include/hepnos/DataSet.hpp new file mode 100644 index 0000000..296d147 --- /dev/null +++ b/include/hepnos/DataSet.hpp @@ -0,0 +1,23 @@ +#ifndef __HEPNOS_DATA_SET_H +#define __HEPNOS_DATA_SET_H + +#include + +namespace hepnos { + +class DataSet { + + friend class DataStore; + + private: + + DataSet(DataStore& ds, uint8_t level, const std::string& name); + + DataStore& m_datastore; + uint8_t m_level; + std::string m_name; +}; + +} + +#endif diff --git a/include/hepnos/DataStore.hpp b/include/hepnos/DataStore.hpp new file mode 100644 index 0000000..d42d3bd --- /dev/null +++ b/include/hepnos/DataStore.hpp @@ -0,0 +1,119 @@ +#ifndef __HEPNOS_DATA_STORE_H +#define __HEPNOS_DATA_STORE_H + +#include +#include + +namespace hepnos { + +class DataSet; + +class DataStore { + + public: + + DataStore(const std::string& configFile); + + DataStore(const DataStore&) = delete; + DataStore(DataStore&&); + DataStore& operator=(const DataStore&) = delete; + DataStore& operator=(DataStore&&); + + ~DataStore(); + + class iterator; + class const_iterator; + + iterator find(const std::string& datasetName); + + inline const_iterator find(const std::string& datasetName) const; + + iterator begin(); + + iterator end(); + + const_iterator cbegin() const; + + const_iterator cend() const; + + iterator lower_bound(const std::string& lb); + + const_iterator lower_bound(const std::string& lb) const; + + iterator upper_bound(const std::string& ub); + + const_iterator upper_bound(const std::string& ub) const; + + DataSet createDataSet(const std::string& name); + + private: + + class Impl; + std::unique_ptr m_impl; +}; + +class DataStore::const_iterator { + + friend class DataStore::Impl; + friend class DataStore; + + protected: + + DataStore* m_datastore; + + const_iterator(DataStore& ds); + + public: + + typedef const_iterator self_type; + typedef DataSet value_type; + typedef DataSet& reference; + typedef DataSet* pointer; + typedef int difference_type; + typedef std::forward_iterator_tag iterator_category; + + virtual ~const_iterator(); + const_iterator(const const_iterator&); + const_iterator(const_iterator&&); + const_iterator& operator=(const const_iterator&); + const_iterator& operator=(const_iterator&&); + + self_type operator++(); + self_type operator++(int); + const reference operator*(); + const pointer operator->(); + bool operator==(const self_type& rhs) const; + bool operator!=(const self_type& rhs) const; +}; + +class DataStore::iterator : public DataStore::const_iterator { + + friend class DataStore::Impl; + friend class DataStore; + + private: + + iterator(DataStore& ds); + + public: + + typedef iterator self_type; + typedef DataSet value_type; + typedef DataSet& reference; + typedef DataSet* pointer; + typedef int difference_type; + typedef std::forward_iterator_tag iterator_category; + + ~iterator(); + iterator(const iterator&); + iterator(iterator&&); + iterator& operator=(const iterator&); + iterator& operator=(iterator&&); + + reference operator*(); + pointer operator->(); +}; + +} + +#endif diff --git a/include/hepnos/Exception.hpp b/include/hepnos/Exception.hpp new file mode 100644 index 0000000..ea0fdec --- /dev/null +++ b/include/hepnos/Exception.hpp @@ -0,0 +1,25 @@ +#ifndef __HEPNOS_EXCEPTION_H +#define __HEPNOS_EXCEPTION_H + +#include +#include + +namespace hepnos { + +class Exception : public std::exception +{ + std::string m_msg; + + public: + + Exception(const std::string& msg) : m_msg(msg){} + + virtual const char* what() const noexcept override + { + return m_msg.c_str(); + } +}; + +} + +#endif diff --git a/include/hepnos/FileDataStore.hpp b/include/hepnos/FileDataStore.hpp deleted file mode 100644 index fd0fbcb..0000000 --- a/include/hepnos/FileDataStore.hpp +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef __HEPNOS_FILE_DATASTORE_H -#define __HEPNOS_FILE_DATASTORE_H - -#include -#include -#include -#include - -namespace hepnos { - -namespace fs = boost::filesystem; - -class FileDataStore { - -public: - - typedef FileNamespace namespace_type; - typedef FileRun run_type; - typedef FileSubRun subrun_type; - typedef FileEvent event_type; - - FileDataStore(const std::string path) - : _path(path) { - if(_path.empty()) _path = "./"; - if(_path.back() != '/') _path += std::string("/"); - fs::create_directories(_path); - fs::create_directory(_path+std::string(".ref")); - } - - namespace_type createNamespace(const std::string& name) { - std::string dir = _path + name; - if(fs::is_directory(dir)) - return FileNamespace(name, dir); - fs::create_directory(dir); - return FileNamespace(name, dir); - } - - namespace_type openNamespace(const std::string& name) { - std::string dir = _path + name; - if(fs::is_directory(dir)) - return FileNamespace(name, dir); - else - return FileNamespace(); - } - - event_type getEventByID(std::uint64_t id) { - std::stringstream ss; - ss << _path << ".ref/" << std::setfill('0') << std::setw(20) << id; - std::string link = ss.str(); - if(!fs::exists(link)) return FileEvent(); - std::string p = fs::canonical(link).string(); - return FileEvent(p); - } - -private: - - std::string _path; -}; - -} - -#endif diff --git a/include/hepnos/FileEvent.hpp b/include/hepnos/FileEvent.hpp deleted file mode 100644 index 21e95e9..0000000 --- a/include/hepnos/FileEvent.hpp +++ /dev/null @@ -1,90 +0,0 @@ -#ifndef __HEPNOS_FILE_EVENT_H -#define __HEPNOS_FILE_EVENT_H - -#include -#include -#include -#include -#include -#include - -namespace hepnos { - -namespace fs = boost::filesystem; - -class FileSubRun; -class FileDataStore; - -class FileEvent : public ProductAccessor { - - private: - - friend class FileDataStore; - friend class FileSubRun; - friend class FileObjectIterator; - - void createRefAndSetID() { - std::size_t h = std::hash()(_path); - std::string linkName; - std::stringstream ss; - do { - ss.str(""); - ss << _path << "../../../../.ref/"; - ss << std::setfill('0') << std::setw(20) << h; - h += 1; - } while(fs::exists(ss.str())); - _eventID = h-1; - fs::create_symlink(_path, ss.str()); - } - - FileEvent(std::uint64_t eventNumber, const std::string& dir) - : ProductAccessor(dir) - , _eventNumber(eventNumber) - , _path(dir) { - createRefAndSetID(); - } - - FileEvent(const std::string& dir) - : ProductAccessor(dir) - , _eventNumber(0) - , _path(dir) { - if(_path.back() != '/') _path += std::string("/"); - std::size_t i,j; - j = _path.size()-1; - if(_path[j] == '/') j--; - i = j; - while(_path[i] != '/') i--; - i += 1; - while(_path[i] == '0') i++; - j += 1; - std::string eventDir(&dir[i], j-i); - if(eventDir.size() > 0) - _eventNumber = std::stoi(eventDir); - } - - FileEvent() - : ProductAccessor("") - , _eventNumber(std::numeric_limits::max()) - , _path("") {} - - std::uint64_t _eventNumber; - std::string _path; - std::size_t _eventID; - - public: - - std::uint64_t getEventNumber() const { - return _eventNumber; - } - - std::size_t getEventID() const { - return _eventID; - } - - bool isValid() const { - return _eventNumber != std::numeric_limits::max(); - } -}; - -} -#endif diff --git a/include/hepnos/FileNamespace.hpp b/include/hepnos/FileNamespace.hpp deleted file mode 100644 index 29787dc..0000000 --- a/include/hepnos/FileNamespace.hpp +++ /dev/null @@ -1,84 +0,0 @@ -#ifndef __HEPNOS_FILE_NAMESPACE_H -#define __HEPNOS_FILE_NAMESPACE_H - -#include -#include -#include -#include -#include - -namespace hepnos { - -namespace fs = boost::filesystem; - -class FileDataStore; - -class FileNamespace { - - friend class FileDataStore; - - private: - - std::string _name; - std::string _path; - - FileNamespace(const std::string& name, const std::string& dir) - : _name(name), _path(dir+std::string("/")) {} - - FileNamespace() {} - - public: - - typedef FileObjectIterator iterator; - - const std::string& getName() const { - return _name; - } - - bool isValid() const { - return !_name.empty(); - } - - FileRun createRun() { - fs::directory_iterator begin(_path), end; - size_t runNumber = std::count_if(begin, end, - [](const fs::directory_entry& d) { - return fs::is_directory(d.path()); - }); - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/"; - std::string dir = ss.str(); - fs::create_directory(dir); - return FileRun(runNumber, dir); - } - - FileRun openRun(std::uint64_t runNumber) { - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << runNumber << "/"; - std::string dir = ss.str(); - if(fs::is_directory(dir)) - return FileRun(runNumber, dir); - else - return FileRun(); - } - - std::size_t numRuns() const { - fs::directory_iterator begin(_path), end; - return std::count_if(begin, end, - [](const fs::directory_entry& d) { - return fs::is_directory(d.path()); - }); - } - - iterator begin() { - return iterator(_path,0,numRuns()); - } - - iterator end() { - auto n = numRuns(); - return iterator(_path,n,n); - } -}; - -} -#endif diff --git a/include/hepnos/FileObjectIterator.hpp b/include/hepnos/FileObjectIterator.hpp deleted file mode 100644 index 662d38b..0000000 --- a/include/hepnos/FileObjectIterator.hpp +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef __HEPNOS_FILE_OBJECT_ITERATOR_H -#define __HEPNOS_FILE_OBJECT_ITERATOR_H - -#include - -namespace hepnos { - -namespace fs = boost::filesystem; - -template -class FileObjectIterator { - - private: - - std::string _path; - std::uint64_t _numObjects; - std::uint64_t _currentIdx; - T _current; - - public: - - FileObjectIterator(const std::string& path, std::uint64_t idx, std::uint64_t nobj) - : _path(path), _numObjects(nobj), _currentIdx(idx) { - if(_path.back() != '/') _path += std::string("/"); - if(_currentIdx != _numObjects) { - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/"; - _current = T(_currentIdx, ss.str()); - } - } - - bool operator==(const FileObjectIterator& other) const { - return _path == other._path - && _currentIdx == other._currentIdx - && _numObjects == other._numObjects; - } - - bool operator!=(const FileObjectIterator& other) const { - return !(*this == other); - } - - const T& operator*() const { - return _current; - } - - const T* operator->() const { - return &_current; - } - - T* operator->() { - return &_current; - } - - FileObjectIterator operator++() { - FileObjectIterator old = *this; - _currentIdx += 1; - if(_currentIdx != _numObjects) { - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/"; - _current = T(_currentIdx, ss.str()); - } - return old; - } - - FileObjectIterator& operator++(int) { - _currentIdx += 1; - if(_currentIdx != _numObjects) { - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << _currentIdx << "/"; - _current = T(_currentIdx, ss.str()); - } - return *this; - } -}; - -} - -#endif diff --git a/include/hepnos/FileProductAccessorBackend.hpp b/include/hepnos/FileProductAccessorBackend.hpp deleted file mode 100644 index e8650b0..0000000 --- a/include/hepnos/FileProductAccessorBackend.hpp +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H -#define __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H - -#include -#include -#include -#include -#include - -namespace hepnos { - -class FileProductAccessorBackend { - - private: - - std::string makeFileNameFor(const std::string& objName, - const InputTag& tag) { - std::string dot("."); - std::string fileName = - objName + dot + tag.moduleLabel + dot - + tag.instanceName + dot + tag.processName; - if(_path.empty()) return fileName; - else if(_path.back() == '/') return _path + fileName; - else return _path + std::string("/") + fileName; - } - - std::string _path; - - public: - - FileProductAccessorBackend(const std::string& path) - : _path(path) {} - - void store(const std::string& objName, - const InputTag& tag, - const std::vector& data) { - std::string fileName = makeFileNameFor(objName, tag); - std::ofstream outfile; - outfile.open(fileName, std::ofstream::out | std::ofstream::trunc); - outfile.write(data.data(), data.size()); - outfile.close(); - } - - bool load(const std::string& objName, - const InputTag& tag, - std::vector& data) { - std::string fileName = makeFileNameFor(objName, tag); - std::ifstream infile; - infile.open(fileName, std::ifstream::ate | std::ifstream::binary); - if(!infile.good()) return false; - std::size_t size = infile.tellg(); - infile.seekg(0, infile.beg); - data.resize(size); - infile.read(data.data(), size); - infile.close(); - return true; - } - - ~FileProductAccessorBackend() {} - -}; - -} -#endif diff --git a/include/hepnos/FileRun.hpp b/include/hepnos/FileRun.hpp deleted file mode 100644 index 569cf8e..0000000 --- a/include/hepnos/FileRun.hpp +++ /dev/null @@ -1,110 +0,0 @@ -#ifndef __HEPNOS_FILE_RUN_H -#define __HEPNOS_FILE_RUN_H - -#include -#include -#include -#include -#include -#include -#include - -namespace hepnos { - -namespace fs = boost::filesystem; - -class FileNamespace; -template class FileObjectIterator; - -class FileRun : public ProductAccessor { - - private: - - friend class FileNamespace; - friend class FileObjectIterator; - - FileRun(std::uint64_t runNumber, const std::string& dir) - : ProductAccessor(dir) - , _runNumber(runNumber) - , _path(dir) {} - - FileRun(const std::string& dir) - : ProductAccessor(dir) - , _runNumber(0) - , _path(dir) { - std::size_t i,j; - j = dir.size()-1; - if(dir[j] == '/') j--; - i = j; - while(dir[i] != '/') i--; - i += 1; - while(dir[i] == '0') i++; - j += 1; - std::string runDir(&dir[i], j-i); - if(runDir.size() > 0) - _runNumber = std::stoi(runDir); - } - - FileRun() - : ProductAccessor("") - , _runNumber(std::numeric_limits::max()) - , _path("") {} - - std::uint64_t _runNumber; - std::string _path; - - public: - - typedef FileObjectIterator iterator; - - std::uint64_t getRunNumber() const { - return _runNumber; - } - - bool isValid() const { - return _runNumber != std::numeric_limits::max(); - } - - FileSubRun createSubRun() { - fs::directory_iterator begin(_path), end; - size_t subRunNumber = std::count_if(begin, end, - [](const fs::directory_entry& d) { - return fs::is_directory(d.path()); - }); - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/"; - std::string dir = ss.str(); - fs::create_directory(dir); - return FileSubRun(subRunNumber, dir); - } - - FileSubRun openSubRun(std::uint64_t subRunNumber) { - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << subRunNumber << "/"; - std::string dir = ss.str(); - if(fs::is_directory(dir)) - return FileSubRun(subRunNumber, dir); - else - return FileSubRun(); - } - - std::size_t numSubRuns() const { - fs::directory_iterator begin(_path), end; - return std::count_if(begin, end, - [](const fs::directory_entry& d) { - return fs::is_directory(d.path()); - }); - } - - iterator begin() { - return iterator(_path,0,numSubRuns()); - } - - iterator end() { - auto n = numSubRuns(); - return iterator(_path,n,n); - } -}; - -} -#endif diff --git a/include/hepnos/FileSubRun.hpp b/include/hepnos/FileSubRun.hpp deleted file mode 100644 index 808f4b6..0000000 --- a/include/hepnos/FileSubRun.hpp +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef __HEPNOS_FILE_SUB_RUN_H -#define __HEPNOS_FILE_SUB_RUN_H - -#include -#include -#include -#include -#include -#include - -namespace hepnos { - -namespace fs = boost::filesystem; - -class FileRun; - -class FileSubRun : public ProductAccessor { - - private: - - friend class FileRun; - friend class FileObjectIterator; - - FileSubRun(std::uint64_t subRunNumber, const std::string& dir) - : ProductAccessor(dir) - , _subRunNumber(subRunNumber) - , _path(dir) {} - - FileSubRun() - : ProductAccessor("") - , _subRunNumber(std::numeric_limits::max()) - , _path("") {} - - std::uint64_t _subRunNumber; - std::string _path; - - public: - - typedef FileObjectIterator iterator; - - std::uint64_t getSubRunNumber() const { - return _subRunNumber; - } - - bool isValid() const { - return _subRunNumber != std::numeric_limits::max(); - } - - FileEvent createEvent() { - fs::directory_iterator begin(_path), end; - size_t eventNumber = std::count_if(begin, end, - [](const fs::directory_entry& d) { - return fs::is_directory(d.path()); - }); - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << eventNumber << "/"; - std::string dir = ss.str(); - fs::create_directory(dir); - return FileEvent(eventNumber, dir); - } - - FileEvent openEvent(std::uint64_t eventNumber) { - std::stringstream ss; - ss << _path << std::setfill('0') << std::setw(12) << eventNumber << "/"; - std::string dir = ss.str(); - if(fs::is_directory(dir)) - return FileEvent(eventNumber, dir); - else - return FileEvent(); - } - - std::size_t numEvents() const { - fs::directory_iterator begin(_path), end; - return std::count_if(begin, end, - [](const fs::directory_entry& d) { - return fs::is_directory(d.path()); - }); - } - - iterator begin() { - return iterator(_path, 0, numEvents()); - } - - iterator end() { - auto n = numEvents(); - return iterator(_path, n, n); - } -}; - -} -#endif diff --git a/include/hepnos/InputTag.hpp b/include/hepnos/InputTag.hpp deleted file mode 100644 index 3403cb8..0000000 --- a/include/hepnos/InputTag.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef __HEPNOS_INPUT_TAG_H -#define __HEPNOS_INPUT_TAG_H - -#include - -namespace hepnos { - -struct InputTag { - - std::string moduleLabel; - std::string instanceName; - std::string processName; - - InputTag(const std::string& ml, - const std::string& in, - const std::string& pn) - : moduleLabel(ml), - instanceName(in), - processName(pn) {} -}; - -} - -#endif diff --git a/include/hepnos/ProductAccessor.hpp b/include/hepnos/ProductAccessor.hpp deleted file mode 100644 index 754e779..0000000 --- a/include/hepnos/ProductAccessor.hpp +++ /dev/null @@ -1,68 +0,0 @@ -#ifndef __HEPNOS_PRODUCT_ACCESSOR_H -#define __HEPNOS_PRODUCT_ACCESSOR_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace hepnos { - -/** - * This function takes an object name created by typeid(obj).name() - * and returns a "friendly" (i.e. compiler-independent) version of - * this name. - */ -static std::string demangle(const char* name) { - int status = -4; - char* res = abi::__cxa_demangle(name, NULL, NULL, &status); - const char* const demangled_name = (status==0)?res:name; - std::string ret_val(demangled_name); - free(res); - return ret_val; -} - -template -class ProductAccessor { - - private: - - BackendProductAccessor _backend; - - public: - - template - ProductAccessor(Args&& ... args) - : _backend(std::forward(args)...) {} - - template - void store(const InputTag& tag, const T& obj) { - std::string objType = demangle(typeid(obj).name()); - std::stringstream ss; - boost::archive::binary_oarchive oa(ss); - oa << obj; - std::string serialized = ss.str(); - std::vector buffer(serialized.begin(), serialized.end()); - _backend.store(objType, tag, buffer); - } - - template - bool load(const InputTag& tag, T& obj) { - std::string objType = demangle(typeid(obj).name()); - std::vector buffer; - if(!_backend.load(objType, tag, buffer)) return false; - std::string serialized(buffer.begin(), buffer.end()); - std::stringstream ss(serialized); - boost::archive::binary_iarchive ia(ss); - ia >> obj; - return true; - } -}; - -} -#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 212e896..a173df2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,5 @@ # list of source files -set(hepnos-server-src hepnos-server.c) +set(hepnos-src DataStore.cpp DataSet.cpp) # load package helper for generating cmake CONFIG packages include (CMakePackageConfigHelpers) @@ -16,9 +16,9 @@ set (HEPNOS_VERSION_PATCH 0) set (hepnos-vers "${HEPNOS_VERSION_MAJOR}.${HEPNOS_VERSION_MINOR}") set (HEPNOS_VERSION "${hepnos-vers}.${HEPNOS_VERSION_PATCH}") -#add_library(hepnos ${hepnos-src}) -#target_link_libraries (hepnos mercury margo) -#target_include_directories (hepnos PUBLIC $) +add_library(hepnos ${hepnos-src}) +target_link_libraries (hepnos mercury margo yaml-cpp sdskv-client bake-client ch-placement) +target_include_directories (hepnos PUBLIC $) # local include's BEFORE, in case old incompatable .h files in prefix/include #target_include_directories (hepnos BEFORE PUBLIC diff --git a/src/DataSet.cpp b/src/DataSet.cpp new file mode 100644 index 0000000..1263b71 --- /dev/null +++ b/src/DataSet.cpp @@ -0,0 +1,10 @@ +#include "hepnos/DataSet.hpp" + +namespace hepnos { + +DataSet::DataSet(DataStore& ds, uint8_t level, const std::string& name) +: m_datastore(ds) +, m_level(level) +, m_name(name) {} + +} diff --git a/src/DataStore.cpp b/src/DataStore.cpp new file mode 100644 index 0000000..f90b2cb --- /dev/null +++ b/src/DataStore.cpp @@ -0,0 +1,388 @@ +#include +#include +#include +#include +#include +#include +#include +#include "private/KeyTypes.hpp" +#include "hepnos/Exception.hpp" +#include "hepnos/DataStore.hpp" +#include "hepnos/DataSet.hpp" + +namespace hepnos { + +class DataStore::Impl { + public: + + DataStore& m_parent; + margo_instance_id m_mid; + sdskv_client_t m_sdskv_client; + bake_client_t m_bake_client; + std::vector m_sdskv_ph; + std::vector m_sdskv_db; // XXX to fill + struct ch_placement_instance* m_chi_sdskv; + std::vector m_bake_ph; + struct ch_placement_instance* m_chi_bake; + const DataStore::iterator m_end; + + Impl(DataStore& parent) + : m_parent(parent) + , m_mid(MARGO_INSTANCE_NULL) + , m_sdskv_client(SDSKV_CLIENT_NULL) + , m_chi_sdskv(nullptr) + , m_bake_client(BAKE_CLIENT_NULL) + , m_chi_bake(nullptr) + , m_end(parent) {} + + void init(const std::string& configFile) { + int ret; + hg_return_t hret; + YAML::Node config = YAML::LoadFile(configFile); + checkConfig(config); + // get protocol + std::string proto = config["hepnos"]["client"]["protocol"].as(); + // initialize Margo + m_mid = margo_init(proto.c_str(), MARGO_CLIENT_MODE, 0, 0); + if(!m_mid) { + cleanup(); + throw Exception("Could not initialized Margo"); + } + // initialize SDSKV client + ret = sdskv_client_init(m_mid, &m_sdskv_client); + if(ret != SDSKV_SUCCESS) { + cleanup(); + throw Exception("Could not create SDSKV client"); + } + // initialize BAKE client + ret = bake_client_init(m_mid, &m_bake_client); + if(ret != 0) { + cleanup(); + throw Exception("Could not create BAKE client"); + } + // create list of sdskv provider handles + YAML::Node sdskv = config["hepnos"]["providers"]["sdskv"]; + for(YAML::const_iterator it = sdskv.begin(); it != sdskv.end(); it++) { + std::string str_addr = it->first.as(); + hg_addr_t addr; + hret = margo_addr_lookup(m_mid, str_addr.c_str(), &addr); + if(hret != HG_SUCCESS) { + margo_addr_free(m_mid,addr); + cleanup(); + throw Exception("margo_addr_lookup failed"); + } + if(it->second.IsScalar()) { + uint16_t provider_id = it->second.as(); + sdskv_provider_handle_t ph; + ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph); + margo_addr_free(m_mid, addr); + if(ret != SDSKV_SUCCESS) { + cleanup(); + throw Exception("sdskv_provider_handle_create failed"); + } + m_sdskv_ph.push_back(ph); + } else if(it->second.IsSequence()) { + for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) { + uint16_t provider_id = pid->second.as(); + sdskv_provider_handle_t ph; + ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph); + margo_addr_free(m_mid, addr); + if(ret != SDSKV_SUCCESS) { + cleanup(); + throw Exception("sdskv_provider_handle_create failed"); + } + m_sdskv_ph.push_back(ph); + } + } + } + // initialize ch-placement for the SDSKV providers + m_chi_sdskv = ch_placement_initialize("hash_lookup3", m_sdskv_ph.size(), 4, 0); + // get list of bake provider handles + YAML::Node bake = config["hepnos"]["providers"]["bake"]; + for(YAML::const_iterator it = bake.begin(); it != bake.end(); it++) { + std::string str_addr = it->first.as(); + hg_addr_t addr; + hret = margo_addr_lookup(m_mid, str_addr.c_str(), &addr); + if(hret != HG_SUCCESS) { + margo_addr_free(m_mid, addr); + cleanup(); + throw Exception("margo_addr_lookup failed"); + } + if(it->second.IsScalar()) { + uint16_t provider_id = it->second.as(); + bake_provider_handle_t ph; + ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph); + margo_addr_free(m_mid, addr); + if(ret != 0) { + cleanup(); + throw Exception("bake_provider_handle_create failed"); + } + m_bake_ph.push_back(ph); + } else if(it->second.IsSequence()) { + for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) { + uint16_t provider_id = pid->second.as(); + bake_provider_handle_t ph; + ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph); + margo_addr_free(m_mid, addr); + if(ret != 0) { + cleanup(); + throw Exception("bake_provider_handle_create failed"); + } + m_bake_ph.push_back(ph); + } + } + } + // initialize ch-placement for the bake providers + if(m_bake_ph.size()) { + m_chi_bake = ch_placement_initialize("hash_lookup3", m_bake_ph.size(), 4, 0); + } + } + + void cleanup() { + for(auto ph : m_sdskv_ph) { + sdskv_provider_handle_release(ph); + } + for(auto ph : m_bake_ph) { + bake_provider_handle_release(ph); + } + sdskv_client_finalize(m_sdskv_client); + bake_client_finalize(m_bake_client); + ch_placement_finalize(m_chi_sdskv); + ch_placement_finalize(m_chi_bake); + if(m_mid) margo_finalize(m_mid); + } + + private: + + static void checkConfig(YAML::Node& config) { + auto hepnosNode = config["hepnos"]; + if(!hepnosNode) { + throw Exception("\"hepnos\" entry not found in YAML file"); + } + auto clientNode = hepnosNode["client"]; + if(!clientNode) { + throw Exception("\"client\" entry not found in \"hepnos\" section"); + } + auto protoNode = clientNode["protocol"]; + if(!protoNode) { + throw Exception("\"protocol\" entry not found in \"client\" section"); + } + auto providersNode = hepnosNode["providers"]; + if(!providersNode) { + throw Exception("\"providers\" entry not found in \"hepnos\" section"); + } + auto sdskvNode = providersNode["sdskv"]; + if(!sdskvNode) { + throw Exception("\"sdskv\" entry not found in \"providers\" section"); + } + if(sdskvNode.size() == 0) { + throw Exception("No provider found in \"sdskv\" section"); + } + for(auto it = sdskvNode.begin(); it != sdskvNode.end(); it++) { + if(it->second.IsScalar()) continue; // one provider id given + if(it->second.IsSequence()) { // array of provider ids given + if(it->second.size() == 0) { + throw Exception("Empty array of provider ids encountered in \"sdskv\" section"); + } + for(auto pid = it->second.begin(); pid != it->second.end(); pid++) { + if(!pid->second.IsScalar()) { + throw Exception("Non-scalar provider id encountered in \"sdskv\" section"); + } + } + } else { + throw Exception("Invalid value type for provider in \"sdskv\" section"); + } + } + // bake providers are not mandatory. If they are not present, + // objects will be stored in sdskv providers. + auto bakeNode = providersNode["bake"]; + if(!bakeNode) return; + if(bakeNode.size() == 0) return; + for(auto it = bakeNode.begin(); it != bakeNode.end(); it++) { + if(it->second.IsScalar()) continue; // one provider id given + if(it->second.IsSequence()) { // array of provider ids given + if(it->second.size() == 0) { + throw Exception("No provider found in \"bake\" section"); + } + for(auto pid = it->second.begin(); pid != it->second.end(); pid++) { + if(!pid->second.IsScalar()) { + throw Exception("Non-scalar provider id encountered in \"bake\" section"); + } + } + } else { + throw Exception("Invalid value type for provider in \"bake\" section"); + } + } + } +}; + +DataStore::DataStore(const std::string& configFile) +: m_impl(std::make_unique(*this)) { + m_impl->init(configFile); +} + +DataStore::DataStore(DataStore&& other) +: m_impl(std::move(other.m_impl)) {} + +DataStore& DataStore::operator=(DataStore&& other) { + if(&other == this) return *this; + if(m_impl) { + m_impl->cleanup(); + } + m_impl = std::move(other.m_impl); +} + +DataStore::~DataStore() { + if(m_impl) { + m_impl->cleanup(); + } +} + +DataStore::iterator DataStore::find(const std::string& datasetName) { + // TODO +} + +DataStore::const_iterator DataStore::find(const std::string& datasetName) const { + // TODO +} + +DataStore::iterator DataStore::begin() { + return iterator(*this); +} + +DataStore::iterator DataStore::end() { + return m_impl->m_end; +} + +DataStore::const_iterator DataStore::cbegin() const { + return const_iterator(const_cast(*this)); +} + +DataStore::const_iterator DataStore::cend() const { + return m_impl->m_end; +} + +DataStore::iterator DataStore::lower_bound(const std::string& lb) { + // TODO +} + +DataStore::const_iterator DataStore::lower_bound(const std::string& lb) const { + // TODO +} + +DataStore::iterator DataStore::upper_bound(const std::string& ub) { + // TODO +} + +DataStore::const_iterator DataStore::upper_bound(const std::string& ub) const { + // TODO +} + +DataSet DataStore::createDataSet(const std::string& name) { + if(name.find('/') != std::string::npos) { + throw Exception("Invalid character '/' in dataset name"); + } + DataStoreEntryPtr entry = make_datastore_entry(0, name); + // find which sdskv provider to contact + uint64_t h = std::hash()(name); + unsigned long provider_idx; + ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx); + // store the key + auto ph = m_impl->m_sdskv_ph[provider_idx]; + auto db_id = m_impl->m_sdskv_db[provider_idx]; + int ret = sdskv_put(ph, db_id, entry->raw(), entry->length(), NULL, 0); + if(ret != SDSKV_SUCCESS) { + throw Exception("Could not create DataSet (sdskv error)"); + } + return DataSet(*this, 0, name); +} + +DataStore::const_iterator::const_iterator(DataStore& ds) +: m_datastore(&ds) { + // TODO +} + +DataStore::const_iterator::~const_iterator() { + // TODO +} + +DataStore::const_iterator::const_iterator(const DataStore::const_iterator& other) +: m_datastore(other.m_datastore) { + // TODO +} + +DataStore::const_iterator::const_iterator(DataStore::const_iterator&& other) +: m_datastore(other.m_datastore) { + // TODO +} + +DataStore::const_iterator& DataStore::const_iterator::operator=(const DataStore::const_iterator&) { + // TODO +} + +DataStore::const_iterator& DataStore::const_iterator::operator=(DataStore::const_iterator&&) { + // TODO +} + +DataStore::const_iterator::self_type DataStore::const_iterator::operator++() { + // TODO +} + +DataStore::const_iterator::self_type DataStore::const_iterator::operator++(int) { + // TODO +} + +const DataStore::const_iterator::reference DataStore::const_iterator::operator*() { + // TODO +} + +const DataStore::const_iterator::pointer DataStore::const_iterator::operator->() { + // TODO +} + +bool DataStore::const_iterator::operator==(const self_type& rhs) const { + // TODO +} + +bool DataStore::const_iterator::operator!=(const self_type& rhs) const { + // TODO +} + +DataStore::iterator::iterator(DataStore& ds) +: const_iterator(ds) { + // TODO +} + +DataStore::iterator::~iterator() { + // TODO +} + +DataStore::iterator::iterator(const DataStore::iterator& other) +: const_iterator(other) { + // TODO +} + +DataStore::iterator::iterator(DataStore::iterator&& other) +: const_iterator(std::move(other)) { + // TODO +} + +DataStore::iterator& DataStore::iterator::operator=(const DataStore::iterator& other) { + m_datastore = other.m_datastore; + // TODO +} + +DataStore::iterator& DataStore::iterator::operator=(DataStore::iterator&& other) { + m_datastore = other.m_datastore; +} + +DataStore::iterator::reference DataStore::iterator::operator*() { + // TODO +} + +DataStore::iterator::pointer DataStore::iterator::operator->() { + // TODO +} + +} + diff --git a/src/private/KeyTypes.hpp b/src/private/KeyTypes.hpp new file mode 100644 index 0000000..3693aae --- /dev/null +++ b/src/private/KeyTypes.hpp @@ -0,0 +1,47 @@ +#ifndef __PRIVATE_KEY_TYPES_H +#define __PRIVATE_KEY_TYPES_H + +#include +#include + +namespace hepnos { + +struct DataStoreEntry { + uint8_t m_level; + char m_fullname[1]; + + DataStoreEntry() = delete; + ~DataStoreEntry() = delete; + + size_t length() const { + return sizeof(*this) + strlen(m_fullname); + } + + const void* raw() const { + return static_cast(this); + } + + void* raw() { + return static_cast(this); + } +}; + +struct DataStoreEntryDeleter { + void operator()(DataStoreEntry* entry) { + free(entry); + } +}; + +typedef std::unique_ptr DataStoreEntryPtr; + +inline DataStoreEntryPtr make_datastore_entry(uint8_t level, const std::string& name) { + size_t s = sizeof(DataStoreEntry) + name.size(); + DataStoreEntry* entry = static_cast(malloc(s)); + entry->m_level = level; + strcpy(entry->m_fullname, name.c_str()); + return DataStoreEntryPtr(entry, DataStoreEntryDeleter()); +} + +} + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..1f2d1a8 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(example example.cpp) +target_link_libraries(example hepnos) diff --git a/test/config.yml b/test/config.yml new file mode 100644 index 0000000..18c5c01 --- /dev/null +++ b/test/config.yml @@ -0,0 +1,6 @@ +hepnos: + client: + protocol: tcp + providers: + sdskv: + tcp://127.0.0.1:1234: 1 diff --git a/test/example.cpp b/test/example.cpp new file mode 100644 index 0000000..48aa11d --- /dev/null +++ b/test/example.cpp @@ -0,0 +1,11 @@ +#include +#include + +using namespace hepnos; + +int main(int argc, char** argv) { + + DataStore ds(argv[1]); + + return 0; +} -- 2.26.2