Skip to content

GitLab

Commit e4b3a9c1 authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

completely moved private function in implementation classes

parent 46b3a0ef
Hide whitespace changes
Inline Side-by-side
Showing with 394 additions and 294 deletions
include/hepnos/DataStore.hpp
View file @ e4b3a9c1
......@@ -214,62 +214,10 @@ class DataStore {
*/
class Impl;
std::unique_ptr<Impl> m_impl; /*!< Pointer to implementation */
/**
* @brief Function used by the DataSet, Run, SubRuns, and Event
* classes to load binary data associated with a particular object.
*
* @param level Level of nesting (0 for data products)
* @param containerName Name of the container
* @param objectName Name of the object
* @param data Resulting byte buffer containing the object's data
*
* @return true if the object was found an loaded correctly,
* false otherwise.
*/
bool load(uint8_t level, const std::string& containerName,
const std::string& objectName, std::vector<char>& data) const;
/**
* @brief Function used by the DataSet, Run, SubRuns, and Event
* classes to load binary data associated with a particular object.
*
* @param level Level of nesting (0 for data products)
* @param containerName Name of the container
* @param objectName Name of the object
* @param data Byte buffer to write
*
* @return true is the object did not exist and was correctly written,
* false otherwise.
*/
bool store(uint8_t level, const std::string& containerName,
const std::string& objectName, const std::vector<char>& data);
/**
* @brief Given a lowerBound for dataset names, retrieve at most
* maxDataSets dataset names coming striclty after lowerBound
* from the DataStore, for a given level and container name.
* Returns the number of dataset names actually retrieved
*
* @param level Level of nesting
* @param containerName Name of the container in which to search
* @param lowerBound Lower bound name to search for
* @param keys Resulting vector of dataset names
* @param maxDataSets Maximum number of datasets to retrieve.
*
* @return The actual number of dataset names retrieved.
*/
size_t nextKeys(uint8_t level, const std::string& containerName,
const std::string& lowerBound,
std::vector<std::string>& keys, size_t maxDataSets) const;
};
class DataStore::const_iterator {
friend class DataStore::Impl;
friend class DataStore;
friend class DataSet;
protected:
/**
......@@ -278,6 +226,7 @@ class DataStore::const_iterator {
class Impl;
std::unique_ptr<Impl> m_impl; /*!< Pointer to implementation */
public:
/**
* @brief Constructor. Creates a const_iterator pointing
* to an invalid DataSet.
......@@ -300,8 +249,6 @@ class DataStore::const_iterator {
*/
const_iterator(DataSet&& current);
public:
typedef const_iterator self_type;
typedef DataSet value_type;
typedef DataSet& reference;
......@@ -404,11 +351,7 @@ class DataStore::const_iterator {
class DataStore::iterator : public DataStore::const_iterator {
friend class DataStore::Impl;
friend class DataStore;
friend class DataSet;
private:
public:
/**
* @brief Constructor. Builds an iterator pointing to an
......@@ -434,8 +377,6 @@ class DataStore::iterator : public DataStore::const_iterator {
*/
iterator(DataSet&& current);
public:
typedef iterator self_type;
typedef DataSet value_type;
typedef DataSet& reference;
......
src/CMakeLists.txt
View file @ e4b3a9c1
......@@ -21,13 +21,14 @@ target_link_libraries (hepnos mercury margo yaml-cpp sdskv-client bake-client ch
target_include_directories (hepnos PUBLIC $<INSTALL_INTERFACE:include>)
# local include's BEFORE, in case old incompatable .h files in prefix/include
#target_include_directories (hepnos BEFORE PUBLIC
# $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>)
include_directories (hepnos ${CMAKE_CURRENT_SOURCE_DIR}/private)
target_include_directories (hepnos BEFORE PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>)
# for shared libs, establish the lib version
#set_target_properties (hepnos
# PROPERTIES VERSION ${HEPNOS_VERSION}
# SOVERSION ${HEPNOS_VERSION_MAJOR})
set_target_properties (hepnos
PROPERTIES VERSION ${HEPNOS_VERSION}
SOVERSION ${HEPNOS_VERSION_MAJOR})
#
# installation stuff (packaging and install commands)
......
src/DataSet.cpp
View file @ e4b3a9c1
#include "hepnos/DataSet.hpp"
#include "private/DataStoreImpl.hpp"
namespace hepnos {
......@@ -54,7 +55,7 @@ DataSet DataSet::next() const {
if(!valid()) return DataSet();
std::vector<std::string> keys;
size_t s = m_impl->m_datastore->nextKeys(
size_t s = m_impl->m_datastore->m_impl->nextKeys(
m_impl->m_level, m_impl->m_container, m_impl->m_name, keys, 1);
if(s == 0) return DataSet();
return DataSet(m_impl->m_datastore, m_impl->m_level, m_impl->m_container, keys[0]);
......@@ -70,7 +71,7 @@ bool DataSet::storeBuffer(const std::string& key, const std::vector<char>& buffe
throw Exception("Calling store() on invalid DataSet");
}
// forward the call to the datastore's store function
return m_impl->m_datastore->store(0, fullname(), key, buffer);
return m_impl->m_datastore->m_impl->store(0, fullname(), key, buffer);
}
bool DataSet::loadBuffer(const std::string& key, std::vector<char>& buffer) const {
......@@ -78,7 +79,7 @@ bool DataSet::loadBuffer(const std::string& key, std::vector<char>& buffer) cons
throw Exception("Calling load() on invalid DataSet");
}
// forward the call to the datastore's load function
return m_impl->m_datastore->load(0, fullname(), key, buffer);
return m_impl->m_datastore->m_impl->load(0, fullname(), key, buffer);
}
bool DataSet::operator==(const DataSet& other) const {
......@@ -113,7 +114,7 @@ DataSet DataSet::createDataSet(const std::string& name) {
throw Exception("Invalid character '/' in dataset name");
}
std::string parent = fullname();
m_impl->m_datastore->store(m_impl->m_level+1, parent, name, std::vector<char>());
m_impl->m_datastore->m_impl->store(m_impl->m_level+1, parent, name, std::vector<char>());
return DataSet(m_impl->m_datastore, m_impl->m_level+1, parent, name);
}
......@@ -129,7 +130,7 @@ DataSet::iterator DataSet::find(const std::string& datasetName) {
}
std::vector<char> data;
std::string parent = fullname();
bool b = m_impl->m_datastore->load(m_impl->m_level+1, parent, datasetName, data);
bool b = m_impl->m_datastore->m_impl->load(m_impl->m_level+1, parent, datasetName, data);
if(!b) {
return m_impl->m_datastore->end();
}
......
src/DataStore.cpp
View file @ e4b3a9c1
......@@ -5,231 +5,13 @@
#include <sdskv-client.h>
#include <bake-client.h>
#include <ch-placement.h>
#include "private/KeyTypes.hpp"
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
#include "private/DataStoreImpl.hpp"
namespace hepnos {
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
class DataStore::Impl {
public:
margo_instance_id m_mid; // Margo instance
sdskv_client_t m_sdskv_client; // SDSKV client
bake_client_t m_bake_client; // BAKE client
std::vector<sdskv_provider_handle_t> m_sdskv_ph; // list of SDSKV provider handlers
std::vector<sdskv_database_id_t> m_sdskv_db; // list of SDSKV database ids
struct ch_placement_instance* m_chi_sdskv; // ch-placement instance for SDSKV
std::vector<bake_provider_handle_t> m_bake_ph; // list of BAKE provider handlers
struct ch_placement_instance* m_chi_bake; // ch-placement instance for BAKE
const DataStore::iterator m_end; // iterator for the end() of the DataStore
Impl(DataStore* parent)
: m_mid(MARGO_INSTANCE_NULL)
, m_sdskv_client(SDSKV_CLIENT_NULL)
, m_chi_sdskv(nullptr)
, m_bake_client(BAKE_CLIENT_NULL)
, m_chi_bake(nullptr)
, m_end() {}
void init(const std::string& configFile) {
int ret;
hg_return_t hret;
YAML::Node config = YAML::LoadFile(configFile);
checkConfig(config);
// get protocol
std::string proto = config["hepnos"]["client"]["protocol"].as<std::string>();
// initialize Margo
m_mid = margo_init(proto.c_str(), MARGO_CLIENT_MODE, 0, 0);
if(!m_mid) {
cleanup();
throw Exception("Could not initialized Margo");
}
// initialize SDSKV client
ret = sdskv_client_init(m_mid, &m_sdskv_client);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("Could not create SDSKV client");
}
// initialize BAKE client
ret = bake_client_init(m_mid, &m_bake_client);
if(ret != 0) {
cleanup();
throw Exception("Could not create BAKE client");
}
// create list of sdskv provider handles
YAML::Node sdskv = config["hepnos"]["providers"]["sdskv"];
for(YAML::const_iterator it = sdskv.begin(); it != sdskv.end(); it++) {
std::string str_addr = it->first.as<std::string>();
hg_addr_t addr;
hret = margo_addr_lookup(m_mid, str_addr.c_str(), &addr);
if(hret != HG_SUCCESS) {
margo_addr_free(m_mid,addr);
cleanup();
throw Exception("margo_addr_lookup failed");
}
if(it->second.IsScalar()) {
uint16_t provider_id = it->second.as<uint16_t>();
sdskv_provider_handle_t ph;
ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_provider_handle_create failed");
}
m_sdskv_ph.push_back(ph);
} else if(it->second.IsSequence()) {
for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) {
uint16_t provider_id = pid->second.as<uint16_t>();
sdskv_provider_handle_t ph;
ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_provider_handle_create failed");
}
m_sdskv_ph.push_back(ph);
}
}
}
// loop over sdskv providers and get the database id
for(auto ph : m_sdskv_ph) {
sdskv_database_id_t db_id;
ret = sdskv_open(ph, "hepnosdb", &db_id);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_open failed to open database");
}
m_sdskv_db.push_back(db_id);
}
// initialize ch-placement for the SDSKV providers
m_chi_sdskv = ch_placement_initialize("hash_lookup3", m_sdskv_ph.size(), 4, 0);
// get list of bake provider handles
YAML::Node bake = config["hepnos"]["providers"]["bake"];
for(YAML::const_iterator it = bake.begin(); it != bake.end(); it++) {
std::string str_addr = it->first.as<std::string>();
hg_addr_t addr;
hret = margo_addr_lookup(m_mid, str_addr.c_str(), &addr);
if(hret != HG_SUCCESS) {
margo_addr_free(m_mid, addr);
cleanup();
throw Exception("margo_addr_lookup failed");
}
if(it->second.IsScalar()) {
uint16_t provider_id = it->second.as<uint16_t>();
bake_provider_handle_t ph;
ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != 0) {
cleanup();
throw Exception("bake_provider_handle_create failed");
}
m_bake_ph.push_back(ph);
} else if(it->second.IsSequence()) {
for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) {
uint16_t provider_id = pid->second.as<uint16_t>();
bake_provider_handle_t ph;
ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != 0) {
cleanup();
throw Exception("bake_provider_handle_create failed");
}
m_bake_ph.push_back(ph);
}
}
}
// initialize ch-placement for the bake providers
if(m_bake_ph.size()) {
m_chi_bake = ch_placement_initialize("hash_lookup3", m_bake_ph.size(), 4, 0);
}
}
void cleanup() {
for(auto ph : m_sdskv_ph) {
sdskv_provider_handle_release(ph);
}
for(auto ph : m_bake_ph) {
bake_provider_handle_release(ph);
}
sdskv_client_finalize(m_sdskv_client);
bake_client_finalize(m_bake_client);
if(m_chi_sdskv)
ch_placement_finalize(m_chi_sdskv);
if(m_chi_bake)
ch_placement_finalize(m_chi_bake);
if(m_mid) margo_finalize(m_mid);
}
private:
static void checkConfig(YAML::Node& config) {
auto hepnosNode = config["hepnos"];
if(!hepnosNode) {
throw Exception("\"hepnos\" entry not found in YAML file");
}
auto clientNode = hepnosNode["client"];
if(!clientNode) {
throw Exception("\"client\" entry not found in \"hepnos\" section");
}
auto protoNode = clientNode["protocol"];
if(!protoNode) {
throw Exception("\"protocol\" entry not found in \"client\" section");
}
auto providersNode = hepnosNode["providers"];
if(!providersNode) {
throw Exception("\"providers\" entry not found in \"hepnos\" section");
}
auto sdskvNode = providersNode["sdskv"];
if(!sdskvNode) {
throw Exception("\"sdskv\" entry not found in \"providers\" section");
}
if(sdskvNode.size() == 0) {
throw Exception("No provider found in \"sdskv\" section");
}
for(auto it = sdskvNode.begin(); it != sdskvNode.end(); it++) {
if(it->second.IsScalar()) continue; // one provider id given
if(it->second.IsSequence()) { // array of provider ids given
if(it->second.size() == 0) {
throw Exception("Empty array of provider ids encountered in \"sdskv\" section");
}
for(auto pid = it->second.begin(); pid != it->second.end(); pid++) {
if(!pid->second.IsScalar()) {
throw Exception("Non-scalar provider id encountered in \"sdskv\" section");
}
}
} else {
throw Exception("Invalid value type for provider in \"sdskv\" section");
}
}
// bake providers are not mandatory. If they are not present,
// objects will be stored in sdskv providers.
auto bakeNode = providersNode["bake"];
if(!bakeNode) return;
if(bakeNode.size() == 0) return;
for(auto it = bakeNode.begin(); it != bakeNode.end(); it++) {
if(it->second.IsScalar()) continue; // one provider id given
if(it->second.IsSequence()) { // array of provider ids given
if(it->second.size() == 0) {
throw Exception("No provider found in \"bake\" section");
}
for(auto pid = it->second.begin(); pid != it->second.end(); pid++) {
if(!pid->second.IsScalar()) {
throw Exception("Non-scalar provider id encountered in \"bake\" section");
}
}
} else {
throw Exception("Invalid value type for provider in \"bake\" section");
}
}
}
};
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore implementation
////////////////////////////////////////////////////////////////////////////////////////////
......@@ -263,7 +45,7 @@ DataStore::iterator DataStore::find(const std::string& datasetName) {
throw Exception("Invalid character ('/' or '%') in dataset name");
}
std::vector<char> data;
bool b = load(1, "", datasetName, data);
bool b = m_impl->load(1, "", datasetName, data);
if(!b) {
return m_impl->m_end;
}
......@@ -337,10 +119,10 @@ DataSet DataStore::createDataSet(const std::string& name) {
|| name.find('%') != std::string::npos) {
throw Exception("Invalid character ('/' or '%') in dataset name");
}
store(1, "", name, std::vector<char>());
m_impl->store(1, "", name, std::vector<char>());
return DataSet(this, 1, "", name);
}
#if 0
bool DataStore::load(uint8_t level, const std::string& containerName,
const std::string& objectName, std::vector<char>& data) const {
int ret;
......@@ -480,7 +262,7 @@ size_t DataStore::nextKeys(uint8_t level, const std::string& containerName,
// set the resulting keys
return max_keys;
}
#endif
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
......
src/private/DataStoreImpl.hpp 0 → 100644
View file @ e4b3a9c1
#include <vector>
#include <functional>
#include <iostream>
#include <yaml-cpp/yaml.h>
#include <sdskv-client.h>
#include <bake-client.h>
#include <ch-placement.h>
#include "KeyTypes.hpp"
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
namespace hepnos {
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
class DataStore::Impl {
public:
margo_instance_id m_mid; // Margo instance
sdskv_client_t m_sdskv_client; // SDSKV client
bake_client_t m_bake_client; // BAKE client
std::vector<sdskv_provider_handle_t> m_sdskv_ph; // list of SDSKV provider handlers
std::vector<sdskv_database_id_t> m_sdskv_db; // list of SDSKV database ids
struct ch_placement_instance* m_chi_sdskv; // ch-placement instance for SDSKV
std::vector<bake_provider_handle_t> m_bake_ph; // list of BAKE provider handlers
struct ch_placement_instance* m_chi_bake; // ch-placement instance for BAKE
const DataStore::iterator m_end; // iterator for the end() of the DataStore
Impl(DataStore* parent)
: m_mid(MARGO_INSTANCE_NULL)
, m_sdskv_client(SDSKV_CLIENT_NULL)
, m_chi_sdskv(nullptr)
, m_bake_client(BAKE_CLIENT_NULL)
, m_chi_bake(nullptr)
, m_end() {}
void init(const std::string& configFile) {
int ret;
hg_return_t hret;
YAML::Node config = YAML::LoadFile(configFile);
checkConfig(config);
// get protocol
std::string proto = config["hepnos"]["client"]["protocol"].as<std::string>();
// initialize Margo
m_mid = margo_init(proto.c_str(), MARGO_CLIENT_MODE, 0, 0);
if(!m_mid) {
cleanup();
throw Exception("Could not initialized Margo");
}
// initialize SDSKV client
ret = sdskv_client_init(m_mid, &m_sdskv_client);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("Could not create SDSKV client");
}
// initialize BAKE client
ret = bake_client_init(m_mid, &m_bake_client);
if(ret != 0) {
cleanup();
throw Exception("Could not create BAKE client");
}
// create list of sdskv provider handles
YAML::Node sdskv = config["hepnos"]["providers"]["sdskv"];
for(YAML::const_iterator it = sdskv.begin(); it != sdskv.end(); it++) {
std::string str_addr = it->first.as<std::string>();
hg_addr_t addr;
hret = margo_addr_lookup(m_mid, str_addr.c_str(), &addr);
if(hret != HG_SUCCESS) {
margo_addr_free(m_mid,addr);
cleanup();
throw Exception("margo_addr_lookup failed");
}
if(it->second.IsScalar()) {
uint16_t provider_id = it->second.as<uint16_t>();
sdskv_provider_handle_t ph;
ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_provider_handle_create failed");
}
m_sdskv_ph.push_back(ph);
} else if(it->second.IsSequence()) {
for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) {
uint16_t provider_id = pid->second.as<uint16_t>();
sdskv_provider_handle_t ph;
ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_provider_handle_create failed");
}
m_sdskv_ph.push_back(ph);
}
}
}
// loop over sdskv providers and get the database id
for(auto ph : m_sdskv_ph) {
sdskv_database_id_t db_id;
ret = sdskv_open(ph, "hepnosdb", &db_id);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_open failed to open database");
}
m_sdskv_db.push_back(db_id);
}
// initialize ch-placement for the SDSKV providers
m_chi_sdskv = ch_placement_initialize("hash_lookup3", m_sdskv_ph.size(), 4, 0);
// get list of bake provider handles
YAML::Node bake = config["hepnos"]["providers"]["bake"];
for(YAML::const_iterator it = bake.begin(); it != bake.end(); it++) {
std::string str_addr = it->first.as<std::string>();
hg_addr_t addr;
hret = margo_addr_lookup(m_mid, str_addr.c_str(), &addr);
if(hret != HG_SUCCESS) {
margo_addr_free(m_mid, addr);
cleanup();
throw Exception("margo_addr_lookup failed");
}
if(it->second.IsScalar()) {
uint16_t provider_id = it->second.as<uint16_t>();
bake_provider_handle_t ph;
ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != 0) {
cleanup();
throw Exception("bake_provider_handle_create failed");
}
m_bake_ph.push_back(ph);
} else if(it->second.IsSequence()) {
for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) {
uint16_t provider_id = pid->second.as<uint16_t>();
bake_provider_handle_t ph;
ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph);
margo_addr_free(m_mid, addr);
if(ret != 0) {
cleanup();
throw Exception("bake_provider_handle_create failed");
}
m_bake_ph.push_back(ph);
}
}
}
// initialize ch-placement for the bake providers
if(m_bake_ph.size()) {
m_chi_bake = ch_placement_initialize("hash_lookup3", m_bake_ph.size(), 4, 0);
}
}
void cleanup() {
for(auto ph : m_sdskv_ph) {
sdskv_provider_handle_release(ph);
}
for(auto ph : m_bake_ph) {
bake_provider_handle_release(ph);
}
sdskv_client_finalize(m_sdskv_client);
bake_client_finalize(m_bake_client);
if(m_chi_sdskv)
ch_placement_finalize(m_chi_sdskv);
if(m_chi_bake)
ch_placement_finalize(m_chi_bake);
if(m_mid) margo_finalize(m_mid);
}
private:
static void checkConfig(YAML::Node& config) {
auto hepnosNode = config["hepnos"];
if(!hepnosNode) {
throw Exception("\"hepnos\" entry not found in YAML file");
}
auto clientNode = hepnosNode["client"];
if(!clientNode) {