Commit 44f34ae1 authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

added threading options and the possibility to use multiple providers and targets:

parent f7108125
...@@ -30,15 +30,23 @@ namespace hepnos { ...@@ -30,15 +30,23 @@ namespace hepnos {
class DataStore::Impl { class DataStore::Impl {
public: public:
struct database {
sdskv_provider_handle_t m_sdskv_ph;
sdskv_database_id_t m_sdskv_db;
};
struct storage {
bake_provider_handle_t m_bake_ph;
bake_target_id_t m_bake_target;
};
margo_instance_id m_mid; // Margo instance margo_instance_id m_mid; // Margo instance
std::unordered_map<std::string,hg_addr_t> m_addrs; // Addresses used by the service std::unordered_map<std::string,hg_addr_t> m_addrs; // Addresses used by the service
sdskv_client_t m_sdskv_client; // SDSKV client sdskv_client_t m_sdskv_client; // SDSKV client
bake_client_t m_bake_client; // BAKE client bake_client_t m_bake_client; // BAKE client
std::vector<sdskv_provider_handle_t> m_sdskv_ph; // list of SDSKV provider handlers std::vector<database> m_databases; // list of SDSKV databases
std::vector<sdskv_database_id_t> m_sdskv_db; // list of SDSKV database ids
struct ch_placement_instance* m_chi_sdskv; // ch-placement instance for SDSKV struct ch_placement_instance* m_chi_sdskv; // ch-placement instance for SDSKV
std::vector<bake_provider_handle_t> m_bake_ph; // list of BAKE provider handlers std::vector<storage> m_storage; // list of BAKE storage targets
std::vector<bake_target_id_t> m_bake_targets; // list of BAKE target ids
struct ch_placement_instance* m_chi_bake; // ch-placement instance for BAKE struct ch_placement_instance* m_chi_bake; // ch-placement instance for BAKE
const DataStore::iterator m_end; // iterator for the end() of the DataStore const DataStore::iterator m_end; // iterator for the end() of the DataStore
...@@ -91,41 +99,50 @@ class DataStore::Impl { ...@@ -91,41 +99,50 @@ class DataStore::Impl {
} }
m_addrs[str_addr] = addr; m_addrs[str_addr] = addr;
} }
// get the provider id(s) // get the number of providers
if(it->second.IsScalar()) { uint16_t num_providers = it->second.as<uint16_t>();
uint16_t provider_id = it->second.as<uint16_t>(); sdskv_provider_handle_t ph;
sdskv_provider_handle_t ph; for(uint16_t provider_id = 0 ; provider_id < num_providers; provider_id++) {
ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph); ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph);
if(ret != SDSKV_SUCCESS) { if(ret != SDSKV_SUCCESS) {
cleanup(); cleanup();
throw Exception("sdskv_provider_handle_create failed"); throw Exception("sdskv_provider_handle_create failed");
} }
m_sdskv_ph.push_back(ph); size_t db_count = 256;
} else if(it->second.IsSequence()) { ret = sdskv_count_databases(ph, &db_count);
for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) { if(ret != SDSKV_SUCCESS) {
uint16_t provider_id = pid->second.as<uint16_t>(); sdskv_provider_handle_release(ph);
sdskv_provider_handle_t ph; cleanup();
ret = sdskv_provider_handle_create(m_sdskv_client, addr, provider_id, &ph); throw Exception("sdskv_count_databases failed");
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_provider_handle_create failed");
}
m_sdskv_ph.push_back(ph);
} }
std::cerr << "Found " << db_count << " databases" << std::endl;
if(db_count == 0) {
continue;
}
std::vector<sdskv_database_id_t> db_ids(db_count);
std::vector<char*> db_names(db_count);
ret = sdskv_list_databases(ph, &db_count, db_names.data(), db_ids.data());
if(ret != SDSKV_SUCCESS) {
sdskv_provider_handle_release(ph);
cleanup();
throw Exception("sdskv_list_databases failed");
}
std::cout << "db_count is now " << db_count << std::endl;
unsigned i = 0;
for(auto id : db_ids) {
std::cout << "Database: " << id << " " << db_names[i] << std::endl;
database db;
sdskv_provider_handle_ref_incr(ph);
db.m_sdskv_ph = ph;
db.m_sdskv_db = id;
m_databases.push_back(db);
i += 1;
}
sdskv_provider_handle_release(ph);
} }
} }
// loop over sdskv providers and get the database id
for(auto ph : m_sdskv_ph) {
sdskv_database_id_t db_id;
ret = sdskv_open(ph, "hepnosdb", &db_id);
if(ret != SDSKV_SUCCESS) {
cleanup();
throw Exception("sdskv_open failed to open database");
}
m_sdskv_db.push_back(db_id);
}
// initialize ch-placement for the SDSKV providers // initialize ch-placement for the SDSKV providers
m_chi_sdskv = ch_placement_initialize("hash_lookup3", m_sdskv_ph.size(), 4, 0); m_chi_sdskv = ch_placement_initialize("hash_lookup3", m_databases.size(), 4, 0);
// get list of bake provider handles // get list of bake provider handles
YAML::Node bake = config["hepnos"]["providers"]["bake"]; YAML::Node bake = config["hepnos"]["providers"]["bake"];
...@@ -146,54 +163,47 @@ class DataStore::Impl { ...@@ -146,54 +163,47 @@ class DataStore::Impl {
} }
m_addrs[str_addr] = addr; m_addrs[str_addr] = addr;
} }
if(it->second.IsScalar()) { uint16_t num_providers = it->second.as<uint16_t>();
uint16_t provider_id = it->second.as<uint16_t>(); for(uint16_t provider_id = 0; provider_id < num_providers; provider_id++) {
bake_provider_handle_t ph; bake_provider_handle_t ph;
ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph); ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph);
if(ret != 0) { if(ret != 0) {
cleanup(); cleanup();
throw Exception("bake_provider_handle_create failed"); throw Exception("bake_provider_handle_create failed");
} }
m_bake_ph.push_back(ph); uint64_t num_targets;
} else if(it->second.IsSequence()) { std::vector<bake_target_id_t> targets(256);
for(YAML::const_iterator pid = it->second.begin(); pid != it->second.end(); pid++) { ret = bake_probe(ph, 256, targets.data(), &num_targets);
uint16_t provider_id = pid->second.as<uint16_t>(); if(ret != 0) {
bake_provider_handle_t ph; bake_provider_handle_release(ph);
ret = bake_provider_handle_create(m_bake_client, addr, provider_id, &ph); cleanup();
if(ret != 0) { throw Exception("bake_probe failed");
cleanup(); }
throw Exception("bake_provider_handle_create failed"); targets.resize(num_targets);
} for(const auto& id : targets) {
m_bake_ph.push_back(ph); storage tgt;
bake_provider_handle_ref_incr(ph);
tgt.m_bake_ph = ph;
tgt.m_bake_target = id;
m_storage.push_back(tgt);
} }
} // if(it->second.IsSequence()) bake_provider_handle_release(ph);
}
} // for loop } // for loop
// find out the bake targets at each bake provider // find out the bake targets at each bake provider
for(auto& bake_ph : m_bake_ph) {
bake_target_id_t bti;
uint64_t num_targets = 0;
ret = bake_probe(bake_ph, 1, &bti, &num_targets);
if(ret != BAKE_SUCCESS) {
throw Exception("bake_probe failed to retrieve targets");
}
if(num_targets != 1) {
throw Exception("bake_prove returned no target");
}
m_bake_targets.push_back(bti);
}
} }
// initialize ch-placement for the bake providers // initialize ch-placement for the bake providers
if(m_bake_ph.size()) { if(m_storage.size()) {
m_chi_bake = ch_placement_initialize("hash_lookup3", m_bake_ph.size(), 4, 0); m_chi_bake = ch_placement_initialize("hash_lookup3", m_storage.size(), 4, 0);
} }
} }
void cleanup() { void cleanup() {
for(auto ph : m_sdskv_ph) { for(const auto& db : m_databases) {
sdskv_provider_handle_release(ph); sdskv_provider_handle_release(db.m_sdskv_ph);
} }
for(auto ph : m_bake_ph) { for(const auto& tgt : m_storage) {
bake_provider_handle_release(ph); bake_provider_handle_release(tgt.m_bake_ph);
} }
sdskv_client_finalize(m_sdskv_client); sdskv_client_finalize(m_sdskv_client);
bake_client_finalize(m_bake_client); bake_client_finalize(m_bake_client);
...@@ -242,24 +252,7 @@ class DataStore::Impl { ...@@ -242,24 +252,7 @@ class DataStore::Impl {
// for each sdskv entry // for each sdskv entry
for(auto it = sdskvNode.begin(); it != sdskvNode.end(); it++) { for(auto it = sdskvNode.begin(); it != sdskvNode.end(); it++) {
if(it->second.IsScalar()) continue; // one provider id given if(it->second.IsScalar()) continue; // one provider id given
if(it->second.IsSequence()) { // array of provider ids given else {
// the sequence is not empty
if(it->second.size() == 0) {
throw Exception("Empty array of provider ids encountered in \"sdskv\" section");
}
// all objects in the sequence are scalar and appear only once
std::unordered_set<uint16_t> ids;
for(auto pid = it->second.begin(); pid != it->second.end(); pid++) {
if(!pid->second.IsScalar()) {
throw Exception("Non-scalar provider id encountered in \"sdskv\" section");
}
uint16_t pid_int = pid->as<uint16_t>();
if(ids.count(pid_int) != 0) {
throw Exception("Provider id encountered twice in \"sdskv\" section");
}
ids.insert(pid_int);
}
} else {
throw Exception("Invalid value type for provider in \"sdskv\" section"); throw Exception("Invalid value type for provider in \"sdskv\" section");
} }
} }
...@@ -270,22 +263,7 @@ class DataStore::Impl { ...@@ -270,22 +263,7 @@ class DataStore::Impl {
if(bakeNode.size() == 0) return; if(bakeNode.size() == 0) return;
for(auto it = bakeNode.begin(); it != bakeNode.end(); it++) { for(auto it = bakeNode.begin(); it != bakeNode.end(); it++) {
if(it->second.IsScalar()) continue; // one provider id given if(it->second.IsScalar()) continue; // one provider id given
if(it->second.IsSequence()) { // array of provider ids given else {
if(it->second.size() == 0) {
throw Exception("No provider found in \"bake\" section");
}
std::unordered_set<uint16_t> ids;
for(auto pid = it->second.begin(); pid != it->second.end(); pid++) {
if(!pid->second.IsScalar()) {
throw Exception("Non-scalar provider id encountered in \"bake\" section");
}
uint16_t pid_int = pid->as<uint16_t>();
if(ids.count(pid_int) != 0) {
throw Exception("Provider id encountered twice in \"bake\" section");
}
ids.insert(pid_int);
}
} else {
throw Exception("Invalid value type for provider in \"bake\" section"); throw Exception("Invalid value type for provider in \"bake\" section");
} }
} }
...@@ -313,10 +291,11 @@ class DataStore::Impl { ...@@ -313,10 +291,11 @@ class DataStore::Impl {
ch_placement_find_closest(m_chi_sdskv, name_hash, 1, &sdskv_provider_idx); ch_placement_find_closest(m_chi_sdskv, name_hash, 1, &sdskv_provider_idx);
// make corresponding datastore entry // make corresponding datastore entry
DataStoreEntryPtr entry = make_datastore_entry(level, ss.str()); DataStoreEntryPtr entry = make_datastore_entry(level, ss.str());
auto sdskv_ph = m_sdskv_ph[sdskv_provider_idx]; auto& db = m_databases[sdskv_provider_idx];
auto db_id = m_sdskv_db[sdskv_provider_idx]; auto sdskv_ph = db.m_sdskv_ph;
auto db_id = db.m_sdskv_db;
// read the value // read the value
if(level != 0 || m_bake_ph.empty()) { // read directly from sdskv if(level != 0 || m_storage.empty()) { // read directly from sdskv
// find the size of the value, as a way to check if the key exists // find the size of the value, as a way to check if the key exists
hg_size_t vsize; hg_size_t vsize;
...@@ -354,8 +333,9 @@ class DataStore::Impl { ...@@ -354,8 +333,9 @@ class DataStore::Impl {
if(data.size() == 0) return true; if(data.size() == 0) return true;
long unsigned bake_provider_idx = 0; long unsigned bake_provider_idx = 0;
ch_placement_find_closest(m_chi_bake, name_hash, 1, &bake_provider_idx); ch_placement_find_closest(m_chi_bake, name_hash, 1, &bake_provider_idx);
auto bake_ph = m_bake_ph[bake_provider_idx]; auto& bake_info = m_storage[bake_provider_idx];
auto target = m_bake_targets[bake_provider_idx]; auto bake_ph = bake_info.m_bake_ph;
auto target = bake_info.m_bake_target;
uint64_t bytes_read = 0; uint64_t bytes_read = 0;
ret = bake_read(bake_ph, rid_info.getBakeRegionID(), 0, data.data(), data.size(), &bytes_read); ret = bake_read(bake_ph, rid_info.getBakeRegionID(), 0, data.data(), data.size(), &bytes_read);
if(ret != BAKE_SUCCESS) { if(ret != BAKE_SUCCESS) {
...@@ -389,8 +369,9 @@ class DataStore::Impl { ...@@ -389,8 +369,9 @@ class DataStore::Impl {
ch_placement_find_closest(m_chi_sdskv, name_hash, 1, &sdskv_provider_idx); ch_placement_find_closest(m_chi_sdskv, name_hash, 1, &sdskv_provider_idx);
// make corresponding datastore entry key // make corresponding datastore entry key
DataStoreEntryPtr entry = make_datastore_entry(level, ss.str()); DataStoreEntryPtr entry = make_datastore_entry(level, ss.str());
auto sdskv_ph = m_sdskv_ph[sdskv_provider_idx]; const auto& sdskv_info = m_databases[sdskv_provider_idx];
auto db_id = m_sdskv_db[sdskv_provider_idx]; auto sdskv_ph = sdskv_info.m_sdskv_ph;
auto db_id = sdskv_info.m_sdskv_db;
// check if the key exists // check if the key exists
hg_size_t vsize; hg_size_t vsize;
int ret = sdskv_length(sdskv_ph, db_id, entry->raw(), entry->length(), &vsize); int ret = sdskv_length(sdskv_ph, db_id, entry->raw(), entry->length(), &vsize);
...@@ -399,7 +380,7 @@ class DataStore::Impl { ...@@ -399,7 +380,7 @@ class DataStore::Impl {
throw Exception("Could not check if key exists in SDSKV (sdskv_length error)"); throw Exception("Could not check if key exists in SDSKV (sdskv_length error)");
} }
// if it's not a last-level data entry (data product), store in sdskeyval // if it's not a last-level data entry (data product), store in sdskeyval
if(level != 0 || m_bake_ph.empty()) { if(level != 0 || m_storage.empty()) {
ret = sdskv_put(sdskv_ph, db_id, entry->raw(), entry->length(), data.data(), data.size()); ret = sdskv_put(sdskv_ph, db_id, entry->raw(), entry->length(), data.data(), data.size());
if(ret != SDSKV_SUCCESS) { if(ret != SDSKV_SUCCESS) {
throw Exception("Could not put key/value pair in SDSKV (sdskv_put error)"); throw Exception("Could not put key/value pair in SDSKV (sdskv_put error)");
...@@ -407,8 +388,9 @@ class DataStore::Impl { ...@@ -407,8 +388,9 @@ class DataStore::Impl {
} else { // store data in bake } else { // store data in bake
long unsigned bake_provider_idx = 0; long unsigned bake_provider_idx = 0;
ch_placement_find_closest(m_chi_bake, name_hash, 1, &bake_provider_idx); ch_placement_find_closest(m_chi_bake, name_hash, 1, &bake_provider_idx);
auto bake_ph = m_bake_ph[bake_provider_idx]; const auto& bake_info = m_storage[bake_provider_idx];
auto target = m_bake_targets[bake_provider_idx]; auto bake_ph = bake_info.m_bake_ph;
auto target = bake_info.m_bake_target;
bake_region_id_t rid; bake_region_id_t rid;
ret = bake_create_write_persist(bake_ph, target, data.data(), data.size(), &rid); ret = bake_create_write_persist(bake_ph, target, data.data(), data.size(), &rid);
if(ret != BAKE_SUCCESS) { if(ret != BAKE_SUCCESS) {
...@@ -454,8 +436,9 @@ class DataStore::Impl { ...@@ -454,8 +436,9 @@ class DataStore::Impl {
keys_len[i] = sizeof(DataStoreEntry) + 1024; keys_len[i] = sizeof(DataStoreEntry) + 1024;
} }
// get provider and database // get provider and database
auto ph = m_sdskv_ph[provider_idx]; const auto& sdskv_info = m_databases[provider_idx];
auto db_id = m_sdskv_db[provider_idx]; auto ph = sdskv_info.m_sdskv_ph;
auto db_id = sdskv_info.m_sdskv_db;
// issue an sdskv_list_keys // issue an sdskv_list_keys
hg_size_t max_keys = maxKeys; hg_size_t max_keys = maxKeys;
ret = sdskv_list_keys(ph, db_id, lb_entry->raw(), lb_entry->length(), ret = sdskv_list_keys(ph, db_id, lb_entry->raw(), lb_entry->length(),
......
...@@ -5,19 +5,19 @@ ...@@ -5,19 +5,19 @@
namespace hepnos { namespace hepnos {
struct ConnectionInfoGenerator::Impl { struct ConnectionInfoGenerator::Impl {
std::string m_addr; // address of this process std::string m_addr; // address of this process
uint16_t m_bake_id; // provider ids for BAKE uint16_t m_num_bake_providers; // number of BAKE provider ids
uint16_t m_sdskv_id; // provider ids for SDSKV uint16_t m_num_sdskv_providers; // number of SDSKV provider ids
}; };
ConnectionInfoGenerator::ConnectionInfoGenerator( ConnectionInfoGenerator::ConnectionInfoGenerator(
const std::string& address, const std::string& address,
uint16_t sdskv_provider_id, uint16_t sdskv_providers,
uint16_t bake_provider_id) uint16_t bake_providers)
: m_impl(std::make_unique<Impl>()) { : m_impl(std::make_unique<Impl>()) {
m_impl->m_addr = address; m_impl->m_addr = address;
m_impl->m_bake_id = bake_provider_id; m_impl->m_num_bake_providers = bake_providers;
m_impl->m_sdskv_id = sdskv_provider_id; m_impl->m_num_sdskv_providers = sdskv_providers;
} }
ConnectionInfoGenerator::~ConnectionInfoGenerator() {} ConnectionInfoGenerator::~ConnectionInfoGenerator() {}
...@@ -42,7 +42,7 @@ void ConnectionInfoGenerator::generateFile(MPI_Comm comm, const std::string& fil ...@@ -42,7 +42,7 @@ void ConnectionInfoGenerator::generateFile(MPI_Comm comm, const std::string& fil
// Exchange bake providers info // Exchange bake providers info
std::vector<uint16_t> bake_pr_ids_buf(size); std::vector<uint16_t> bake_pr_ids_buf(size);
MPI_Gather(&(m_impl->m_bake_id), MPI_Gather(&(m_impl->m_num_bake_providers),
1, MPI_UNSIGNED_SHORT, 1, MPI_UNSIGNED_SHORT,
bake_pr_ids_buf.data(), bake_pr_ids_buf.data(),
1, MPI_UNSIGNED_SHORT, 1, MPI_UNSIGNED_SHORT,
...@@ -50,7 +50,7 @@ void ConnectionInfoGenerator::generateFile(MPI_Comm comm, const std::string& fil ...@@ -50,7 +50,7 @@ void ConnectionInfoGenerator::generateFile(MPI_Comm comm, const std::string& fil
// Exchange sdskv providers info // Exchange sdskv providers info
std::vector<uint16_t> sdskv_pr_ids_buf(size); std::vector<uint16_t> sdskv_pr_ids_buf(size);
MPI_Gather(&(m_impl->m_sdskv_id), MPI_Gather(&(m_impl->m_num_sdskv_providers),
1, MPI_UNSIGNED_SHORT, 1, MPI_UNSIGNED_SHORT,
sdskv_pr_ids_buf.data(), sdskv_pr_ids_buf.data(),
1, MPI_UNSIGNED_SHORT, 1, MPI_UNSIGNED_SHORT,
......
...@@ -17,8 +17,7 @@ private: ...@@ -17,8 +17,7 @@ private:
public: public:
ConnectionInfoGenerator(const std::string& address, ConnectionInfoGenerator(const std::string& address,
uint16_t sdskv_provider_id, uint16_t num_sdskv_providers, uint16_t num_bake_providers);
uint16_t bake_provider_id);
ConnectionInfoGenerator(const ConnectionInfoGenerator&) = delete; ConnectionInfoGenerator(const ConnectionInfoGenerator&) = delete;
ConnectionInfoGenerator(ConnectionInfoGenerator&&) = delete; ConnectionInfoGenerator(ConnectionInfoGenerator&&) = delete;
ConnectionInfoGenerator& operator=(const ConnectionInfoGenerator&) = delete; ConnectionInfoGenerator& operator=(const ConnectionInfoGenerator&) = delete;
......
...@@ -40,7 +40,7 @@ void hepnos_run_service(MPI_Comm comm, const char* config_file, const char* conn ...@@ -40,7 +40,7 @@ void hepnos_run_service(MPI_Comm comm, const char* config_file, const char* conn
} }
/* Margo initialization */ /* Margo initialization */
mid = margo_init(config->getAddress().c_str(), MARGO_SERVER_MODE, 0, -1); mid = margo_init(config->getAddress().c_str(), MARGO_SERVER_MODE, 0, config->getNumThreads()-1);
if (mid == MARGO_INSTANCE_NULL) if (mid == MARGO_INSTANCE_NULL)
{ {
std::cerr << "Error: unable to initialize margo" << std::endl; std::cerr << "Error: unable to initialize margo" << std::endl;
...@@ -57,54 +57,59 @@ void hepnos_run_service(MPI_Comm comm, const char* config_file, const char* conn ...@@ -57,54 +57,59 @@ void hepnos_run_service(MPI_Comm comm, const char* config_file, const char* conn
hg_size_t self_addr_str_size = 128; hg_size_t self_addr_str_size = 128;
margo_addr_to_string(mid, self_addr_str, &self_addr_str_size, self_addr); margo_addr_to_string(mid, self_addr_str, &self_addr_str_size, self_addr);
uint16_t bake_provider_id = 0;
if(config->hasStorage()) { if(config->hasStorage()) {
/* Bake provider initialization */ for(auto bake_provider_id = 0; bake_provider_id < config->getNumStorageProviders(); bake_provider_id++) {
bake_provider_id = 1; // XXX we can make that come from the config file /* create provider */
const char* bake_target_name = config->getStoragePath().c_str(); bake_provider_t bake_prov;
size_t bake_target_size = config->getStorageSize()*(1024*1024); ret = bake_provider_register(mid, bake_provider_id, BAKE_ABT_POOL_DEFAULT, &bake_prov);
/* create the bake target if it does not exist */ ASSERT(ret == 0, "bake_provider_register() failed (ret = %d)\n", ret);
if(-1 == access(bake_target_name, F_OK)) { /* create databases */
ret = bake_makepool(bake_target_name, bake_target_size, 0664); for(unsigned i=0; i < config->getNumStorageTargets(); i++) {
ASSERT(ret == 0, "bake_makepool() failed (ret = %d)\n", ret); auto bake_target_name = config->getStoragePath(rank, bake_provider_id, i);
size_t bake_target_size = config->getStorageSize()*(1024*1024);
if(-1 == access(bake_target_name.c_str(), F_OK)) {
ret = bake_makepool(bake_target_name.c_str(), bake_target_size, 0664);
ASSERT(ret == 0, "bake_makepool() failed (ret = %d)\n", ret);
}
bake_target_id_t bake_tid;
ret = bake_provider_add_storage_target(bake_prov, bake_target_name.c_str(), &bake_tid);
ASSERT(ret == 0, "bake_provider_add_storage_target() failed to add target %s (ret = %d)\n",
bake_target_name.c_str(), ret);
}
} }
bake_provider_t bake_prov;
bake_target_id_t bake_tid;
ret = bake_provider_register(mid, bake_provider_id, BAKE_ABT_POOL_DEFAULT, &bake_prov);
ASSERT(ret == 0, "bake_provider_register() failed (ret = %d)\n", ret);
ret = bake_provider_add_storage_target(bake_prov, bake_target_name, &bake_tid);
ASSERT(ret == 0, "bake_provider_add_storage_target() failed to add target %s (ret = %d)\n",
bake_target_name, ret);
} }
uint8_t sdskv_provider_id = 0;
if(config->hasDatabase()) { if(config->hasDatabase()) {
/* SDSKV provider initialization */ /* SDSKV provider initialization */
sdskv_provider_id = 2; // XXX we can make that come from the config file for(auto sdskv_provider_id = 0; sdskv_provider_id < config->getNumDatabaseProviders(); sdskv_provider_id++) {
sdskv_provider_t sdskv_prov; sdskv_provider_t sdskv_prov;
ret = sdskv_provider_register(mid, sdskv_provider_id, SDSKV_ABT_POOL_DEFAULT, &sdskv_prov); ret = sdskv_provider_register(mid, sdskv_provider_id, SDSKV_ABT_POOL_DEFAULT, &sdskv_prov);
ASSERT(ret == 0, "sdskv_provider_register() failed (ret = %d)\n", ret); ASSERT(ret == 0, "sdskv_provider_register() failed (ret = %d)\n", ret);
/* creating the database */ for(unsigned i=0 ; i < config->getNumDatabaseTargets(); i++) {
const char* db_path = config->getDatabasePath().c_str(); auto db_path = config->getDatabasePath(rank, sdskv_provider_id, i);
const char* db_name = config->getDatabaseName().c_str(); auto db_name = config->getDatabaseName(rank, sdskv_provider_id, i);
sdskv_db_type_t db_type; sdskv_db_type_t db_type;
if(config->getDatabaseType() == "map") db_type = KVDB_MAP; if(config->getDatabaseType() == "map") db_type = KVDB_MAP;
if(config->getDatabaseType() == "ldb") db_type = KVDB_LEVELDB; if(config->getDatabaseType() == "ldb") db_type = KVDB_LEVELDB;
if(config->getDatabaseType() == "bdb") db_type = KVDB_BERKELEYDB; if(config->getDatabaseType() == "bdb") db_type = KVDB_BERKELEYDB;
sdskv_database_id_t db_id; sdskv_database_id_t db_id;
sdskv_config_t config; sdskv_config_t config;
std::memset(&config, 0, sizeof(config)); std::memset(&config, 0, sizeof(config));
config.db_name = db_name; config.db_name = db_name.c_str();
config.db_path = db_path; config.db_path = db_path.c_str();
config.db_type = db_type; config.db_type = db_type;
ret = sdskv_provider_attach_database(sdskv_prov, &config, &db_id); ret = sdskv_provider_attach_database(sdskv_prov, &config, &db_id);
ASSERT(ret == 0, "sdskv_provider_attach_database() failed (ret = %d)\n", ret); ASSERT(ret == 0, "sdskv_provider_attach_database() failed (ret = %d)\n", ret);
}
}
} }
margo_addr_free(mid, self_addr); margo_addr_free(mid, self_addr);
hepnos::ConnectionInfoGenerator fileGen(self_addr_str, sdskv_provider_id, bake_provider_id); hepnos::ConnectionInfoGenerator fileGen(self_addr_str,
config->getNumDatabaseProviders(),
config->getNumStorageProviders());
fileGen.generateFile(MPI_COMM_WORLD, connection_file); fileGen.generateFile(MPI_COMM_WORLD, connection_file);
margo_wait_for_finalize(mid); margo_wait_for_finalize(mid);
......
#include <cmath>
#include <iomanip>
#include "ServiceConfig.hpp" #include "ServiceConfig.hpp"
#include "hepnos/Exception.hpp" #include "hepnos/Exception.hpp"
#include <yaml-cpp/yaml.h> #include <yaml-cpp/yaml.h>
...@@ -7,41 +9,68 @@ namespace hepnos { ...@@ -7,41 +9,68 @@ namespace hepnos {
struct ServiceConfig::Impl { struct ServiceConfig::Impl {
std::string m_address; std::string m_address;
uint32_t m_numRanks;
bool m_hasDatabase; bool m_hasDatabase;
std</