ServiceConfig.cpp 6.95 KB
Newer Older
1
2
#include <cmath>
#include <iomanip>
3
4
#include <iostream>
#include <unordered_map>
5
6
7
8
9
10
11
#include "ServiceConfig.hpp"
#include "hepnos/Exception.hpp"
#include <yaml-cpp/yaml.h>

namespace hepnos {

static YAML::Node loadAndValidate(const std::string& filename);
12

13
14
15
static std::string formatString(const std::string& str, 
        int rank, int provider, int target,
        int maxRank, int maxProvider, int maxTarget);
16

17
18
19
20
21
static std::vector<ProviderConfig> parseDatabaseEntry(
        const YAML::Node& entry, int rank, int maxRank,
        uint16_t start_provider_id,
        int maxProviders);

22
ServiceConfig::ServiceConfig(const std::string& filename, int rank, int numRanks)
23
{
24
    
25
26
27
28
29
30
    this->numRanks = numRanks;

    YAML::Node config    = loadAndValidate(filename);
    YAML::Node address   = config["address"];
    YAML::Node threads   = config["threads"];
    YAML::Node databases = config["databases"];
Matthieu Dorier's avatar
Matthieu Dorier committed
31
32
    YAML::Node busySpin  = config["busy-spin"];
    if(busySpin) this->busySpin = busySpin.as<bool>();
33
34
35
36
37
38
39
40
41
    if(threads) this->numThreads = threads.as<uint32_t>();
    this->address = address.as<std::string>();
    // Count the total number of providers and targets
    unsigned maxProviders = 0;
    std::vector<std::string> fields = {"datasets", "runs", "subruns", "events", "products"};
    for(const auto& field : fields) {
        uint32_t num_providers = 1;
        if(databases[field]["providers"]) num_providers = databases[field]["providers"].as<uint32_t>();
        maxProviders += num_providers;
42
    }
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
    // DataSet DB
    uint16_t start_provider_id = 0;
    this->datasetProviders = parseDatabaseEntry(databases["datasets"], rank, numRanks, start_provider_id, maxProviders);
    start_provider_id += this->datasetProviders.size();
    // Run DB
    this->runProviders = parseDatabaseEntry(databases["runs"], rank, numRanks, start_provider_id, maxProviders);
    start_provider_id += this->runProviders.size();
    // SubRun DB
    this->subrunProviders = parseDatabaseEntry(databases["subruns"], rank, numRanks, start_provider_id, maxProviders);
    start_provider_id += this->subrunProviders.size();
    // Event DB
    this->eventProviders = parseDatabaseEntry(databases["events"], rank, numRanks, start_provider_id, maxProviders);
    start_provider_id += this->eventProviders.size();
    // Product DB
    this->productProviders = parseDatabaseEntry(databases["products"], rank, numRanks, start_provider_id, maxProviders);
58
59
60
61
}

ServiceConfig::~ServiceConfig() {}

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
static std::vector<ProviderConfig> parseDatabaseEntry(
        const YAML::Node& db, int rank, int maxRanks,
        uint16_t start_provider_id, int maxProviders)
{
    std::unordered_map<std::string, sdskv_db_type_t> sdskv_type_from;
    sdskv_type_from["null"] = KVDB_NULL;
    sdskv_type_from["map"]  = KVDB_MAP;
    sdskv_type_from["ldb"]  = KVDB_LEVELDB;
    sdskv_type_from["bdb"]  = KVDB_BERKELEYDB;

    uint16_t provider_id = start_provider_id;
    auto nameTemplate      = db["name"].as<std::string>();
    auto pathTemplate      = db["path"].as<std::string>();
    auto type              = db["type"].as<std::string>();
    uint32_t num_providers = 1;
    uint32_t num_targets   = 1;
    if(db["providers"]) num_providers = db["providers"].as<uint32_t>();
    if(db["targets"])   num_targets   = db["targets"].as<uint32_t>();
    // populate provider and database config
    std::vector<ProviderConfig> result;
    for(unsigned i = 0; i < num_providers; i++) {
        ProviderConfig pconfig;
        pconfig.provider_id = provider_id;
        for(unsigned j = 0; j < num_targets; j++) {
            DataBaseConfig dbconfig;
            dbconfig.name = formatString(nameTemplate, rank, provider_id, j,
                    maxRanks, maxProviders, num_targets);
            dbconfig.path = formatString(pathTemplate, rank, provider_id, j,
                    maxRanks, maxProviders, num_targets);
            dbconfig.type = sdskv_type_from[type];
            pconfig.databases.push_back(dbconfig);
        }
        provider_id += 1;
        result.push_back(pconfig);
    }
    return result;
98
99
}

100
// Configuration file validation
101
102
103
104
105
static YAML::Node loadAndValidate(const std::string& filename) {
    YAML::Node config = YAML::LoadFile(filename);
    if(!config["address"]) {
        throw Exception("\"address\" field not found in configuration file.");
    }
106
    if(!config["databases"]) {
107
108
        throw Exception("\"database\" field not found in configuration file.");
    }
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    std::vector<std::string> db_fields = { "datasets", "runs", "subruns", "events", "products" };
    for(auto& f : db_fields) {
        if(!config["databases"][f]) {
            throw Exception("\"databases."+f+"\" not found in configuration file.");
        }
        auto db_config = config["databases"][f];
        if(!db_config["path"]) {
            throw Exception("\"databases."+f+".path\" field not found in configuration file.");
        }
        if(!db_config["name"]) {
            throw Exception("\"databases."+f+".name\" field not found in configuration file.");
        }
        if(!db_config["type"]) {
            throw Exception("\"databases."+f+".type\" field not found in configuration file.");
123
        }
124
125
126
127
128
129
        std::string db_type = db_config["type"].as<std::string>();
        if(db_type != "null"
        && db_type != "map"
        && db_type != "ldb"
        && db_type != "bdb") {
            throw Exception("\"databases"+f+".type\" field should be \"null\", \"map\", \"ldb\", or \"bdb\".");
130
131
132
133
134
        }
    }
    return config;
}

135
136
137
static std::string formatString(const std::string& str, 
        int rank, int provider, int target,
        int maxRank=0, int maxProvider=0, int maxTarget=0) {
138
139
    std::string result = str;
    std::stringstream ssrank;
140
141
142
143
144
    ssrank << std::setw(log10(maxRank+1)+1) << std::setfill('0') << rank;
    std::stringstream ssprovider;
    ssprovider << std::setw(log10(maxProvider+1)+1) << std::setfill('0') << provider;
    std::stringstream sstarget;
    sstarget << std::setw(log10(maxTarget+1)+1) << std::setfill('0') << target;
145
    std::string srank = ssrank.str();
146
147
148
    std::string sprovider = ssprovider.str();
    std::string starget = sstarget.str();
    size_t index = 0;
149
150
151
152
153
154
155
156
157
158
    while (true) {
        index = result.find("$RANK", index);
        if (index == std::string::npos) break;
        if(rank >= 0) {
            result.replace(index, 5, srank.c_str());
        } else {
            result.replace(index, 5, "");
        }
        index += 5;
    }
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    index = 0;
    while (true) {
        index = result.find("$PROVIDER", index);
        if (index == std::string::npos) break;
        if(rank >= 0) {
            result.replace(index, 9, sprovider.c_str());
        } else {
            result.replace(index, 9, "");
        }
        index += 9;
    }
    index = 0;
    while (true) {
        index = result.find("$TARGET", index);
        if (index == std::string::npos) break;
        if(rank >= 0) {
            result.replace(index, 7, starget.c_str());
        } else {
            result.replace(index, 7, "");
        }
        index += 7;
    }
181
182
183
184
185
    return result;
}

}