DataStore.cpp 13.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
#include <vector>
#include <functional>
#include <iostream>
#include <yaml-cpp/yaml.h>
#include <sdskv-client.h>
#include <bake-client.h>
#include <ch-placement.h>
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
11
#include "private/DataStoreImpl.hpp"
12
13
14

namespace hepnos {

15
16
17
18
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore implementation
////////////////////////////////////////////////////////////////////////////////////////////

19
DataStore::DataStore(const std::string& configFile) 
20
: m_impl(std::make_unique<DataStore::Impl>(this)) {
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
    m_impl->init(configFile);
}

DataStore::DataStore(DataStore&& other)
: m_impl(std::move(other.m_impl)) {}

DataStore& DataStore::operator=(DataStore&& other) {
    if(&other == this) return *this;
    if(m_impl) {
        m_impl->cleanup();
    }
    m_impl = std::move(other.m_impl);
}
    
DataStore::~DataStore() {
    if(m_impl) {
        m_impl->cleanup();
    }
}

DataStore::iterator DataStore::find(const std::string& datasetName) {
42
    int ret;
43
44
45
    if(datasetName.find('/') != std::string::npos
    || datasetName.find('%') != std::string::npos) {
        throw Exception("Invalid character ('/' or '%') in dataset name");
46
    }
47
    std::vector<char> data;
48
    bool b = m_impl->load(1, "", datasetName, data);
49
50
    if(!b) {
        return m_impl->m_end;
51
    }
52
    return iterator(DataSet(this, 1, datasetName));
53
54
}

Matthieu Dorier's avatar
Matthieu Dorier committed
55
56
57
58
59
DataSet DataStore::operator[](const std::string& datasetName) const {
    auto it = find(datasetName);
    return std::move(*it);
}

60
DataStore::const_iterator DataStore::find(const std::string& datasetName) const {
61
62
    DataStore::iterator it = const_cast<DataStore*>(this)->find(datasetName);
    return it;
63
64
65
}

DataStore::iterator DataStore::begin() {
66
    DataSet ds(this, 1, "", "");
67
    ds = ds.next();
68
    if(ds.valid()) return iterator(std::move(ds));
69
    else return end();
70
71
}

72
73
74
75
76
DataStore::const_iterator DataStore::begin() const {
    iterator it = const_cast<DataStore*>(this)->begin();
    return const_iterator(std::move(it));
}

77
78
79
80
DataStore::iterator DataStore::end() {
    return m_impl->m_end;
}

81
82
83
84
DataStore::const_iterator DataStore::end() const {
    return m_impl->m_end;
}

85
DataStore::const_iterator DataStore::cbegin() const {
86
    return const_cast<DataStore*>(this)->begin();
87
88
89
}

DataStore::const_iterator DataStore::cend() const {
90
    return const_cast<DataStore*>(this)->end();
91
92
93
}

DataStore::iterator DataStore::lower_bound(const std::string& lb) {
94
95
96
97
98
99
100
101
102
    std::string lb2 = lb;
    size_t s = lb2.size();
    lb2[s-1] -= 1; // sdskv_list_keys's start_key is exclusive
    iterator it = find(lb2);
    if(it != end()) {
        // we found something before the specified lower bound
        ++it;
        return it;
    }
103
    DataSet ds(this, 1, "", lb2);
104
105
    ds = ds.next();
    if(!ds.valid()) return end();
106
    else return iterator(std::move(ds));
107
108
109
}

DataStore::const_iterator DataStore::lower_bound(const std::string& lb) const {
110
111
    iterator it = const_cast<DataStore*>(this)->lower_bound(lb);
    return it;
112
113
114
}

DataStore::iterator DataStore::upper_bound(const std::string& ub) {
115
    DataSet ds(this, 1, "", ub);
116
117
    ds = ds.next();
    if(!ds.valid()) return end();
118
    else return iterator(std::move(ds));
119
120
121
}

DataStore::const_iterator DataStore::upper_bound(const std::string& ub) const {
122
123
    iterator it = const_cast<DataStore*>(this)->upper_bound(ub);
    return it;
124
125
126
}

DataSet DataStore::createDataSet(const std::string& name) {
127
128
129
    if(name.find('/') != std::string::npos
    || name.find('%') != std::string::npos) {
        throw Exception("Invalid character ('/' or '%') in dataset name");
130
    }
131
    m_impl->store(1, "", name, std::vector<char>());
132
    return DataSet(this, 1, "", name);
133
}
134
#if 0
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
bool DataStore::load(uint8_t level, const std::string& containerName,
        const std::string& objectName, std::vector<char>& data) const {
    int ret;
    // build full name
    std::stringstream ss;
    if(!containerName.empty())
        ss << containerName << "/";
    ss << objectName;
    // hash the name to get the provider id
    long unsigned provider_idx = 0;
    if(level != 0) {
        uint64_t h = std::hash<std::string>()(containerName);
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    } else {
        // use the complete name for final objects (level 255)
        uint64_t h = std::hash<std::string>()(ss.str());
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    }
    // make corresponding datastore entry
    DataStoreEntryPtr entry = make_datastore_entry(level, ss.str());
155
156
    auto ph = m_impl->m_sdskv_ph[provider_idx];
    auto db_id = m_impl->m_sdskv_db[provider_idx];
157
158
159
160
161
162
163
164
165
    // find the size of the value, as a way to check if the key exists
    hg_size_t vsize;
    std::cerr << "[LOG] load (level=" << (int)level
        << ", container=\"" << containerName << "\", object=\""
        << objectName << "\")" << std::endl;
    ret = sdskv_length(ph, db_id, entry->raw(), entry->length(), &vsize);
    if(ret == SDSKV_ERR_UNKNOWN_KEY) {
        return false;
    }
166
    if(ret != SDSKV_SUCCESS) {
167
        throw Exception("Error occured when calling sdskv_length");
168
    }
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
    // read the value
    data.resize(vsize);
    ret = sdskv_get(ph, db_id, entry->raw(), entry->length(), data.data(), &vsize);
    if(ret != SDSKV_SUCCESS) {
        throw Exception("Error occured when calling sdskv_get");
    }
    return true;
}

bool DataStore::store(uint8_t level, const std::string& containerName,
        const std::string& objectName, const std::vector<char>& data) {
    // build full name
    std::stringstream ss;
    if(!containerName.empty())
        ss << containerName << "/";
    ss << objectName;
    // hash the name to get the provider id
    long unsigned provider_idx = 0;
    if(level != 0) {
        uint64_t h = std::hash<std::string>()(containerName);
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    } else {
        // use the complete name for final objects (level 0)
        uint64_t h = std::hash<std::string>()(ss.str());
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    }
    // make corresponding datastore entry
    DataStoreEntryPtr entry = make_datastore_entry(level, ss.str());
    auto ph = m_impl->m_sdskv_ph[provider_idx];
    auto db_id = m_impl->m_sdskv_db[provider_idx];
    std::cerr << "[LOG] store (level=" << (int)level
        << ", container=\"" << containerName << "\", object=\""
        << objectName << "\")" << std::endl;
    int ret = sdskv_put(ph, db_id, entry->raw(), entry->length(), data.data(), data.size());
    if(ret != SDSKV_SUCCESS) {
        throw Exception("Could not put key/value pair in SDSKV (sdskv_put error)");
    }
    return true;
207
208
}

209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
size_t DataStore::nextKeys(uint8_t level, const std::string& containerName,
        const std::string& lower,
        std::vector<std::string>& keys, size_t maxKeys) const {
    int ret;
    if(level == 0) return 0; // cannot iterate at object level
    // build full name from lower bound key
    std::stringstream ss;
    if(!containerName.empty())
        ss << containerName << "/";
    ss << lower;
    // hash the name to get the provider id
    long unsigned provider_idx = 0;
    uint64_t h = std::hash<std::string>()(containerName);
    ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    // make an entry for the lower bound
    DataStoreEntryPtr lb_entry = make_datastore_entry(level, ss.str());
    // create data structures to receive keys
    std::vector<DataStoreEntryPtr> keys_ent;
    std::vector<void*>             keys_ptr(maxKeys);
    std::vector<hg_size_t>         keys_len(maxKeys);
    for(unsigned i=0; i < maxKeys; i++) {
        keys_ent.push_back(make_datastore_entry(level, 1024));
        keys_ptr[i] = keys_ent[i]->raw();
        keys_len[i] = sizeof(DataStoreEntry) + 1024;
    }
    // get provider and database
    auto ph = m_impl->m_sdskv_ph[provider_idx];
    auto db_id = m_impl->m_sdskv_db[provider_idx];
    // issue an sdskv_list_keys
    hg_size_t max_keys = maxKeys;
    std::cerr << "[LOG] list keys (level=" << (int)level
        << ", container=\"" << containerName << "\", greaterthan=\""
        << lower << "\")" << std::endl;
    ret = sdskv_list_keys(ph, db_id, lb_entry->raw(), lb_entry->length(),
                          keys_ptr.data(), keys_len.data(), &max_keys);
    if(ret != HG_SUCCESS) {
        throw Exception("Error occured when calling sdskv_list_keys");
    }
    unsigned i = max_keys - 1;
    if(max_keys == 0) return 0;
    // remove keys that don't have the same level or the same prefix
    std::string prefix = containerName + "/";
    keys.resize(0);
    for(unsigned i = 0; i < max_keys; i++) {
        if(keys_ent[i]->m_level != level) {
            max_keys = i;
            break;
        }
        if(!containerName.empty()) {
            size_t lenpre = prefix.size();
            size_t lenstr = strlen(keys_ent[i]->m_fullname);
            if(lenstr < lenpre) {
                max_keys = i;
                break;
            }
            if(strncmp(prefix.c_str(), keys_ent[i]->m_fullname, lenpre) != 0) {
                max_keys = i;
                break;
            }
        }
        keys.push_back(keys_ent[i]->m_fullname);
    }
    // set the resulting keys
    return max_keys;
}
274
#endif
275
276
277
278
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////

279
280
281
282
class DataStore::const_iterator::Impl {
    public:
        DataSet    m_current_dataset;

283
284
        Impl()
        : m_current_dataset()
285
        {}
286

287
288
289
290
291
292
        Impl(const DataSet& dataset)
        : m_current_dataset(dataset)
        {}

        Impl(DataSet&& dataset)
        : m_current_dataset(std::move(dataset))
293
        {}
294
295

        Impl(const Impl& other)
296
        : m_current_dataset(other.m_current_dataset) 
297
298
299
        {}

        bool operator==(const Impl& other) const {
300
            return m_current_dataset == other.m_current_dataset;
301
        }
302
303
};

304
305
306
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
307

308
309
DataStore::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}
310

311
312
313
314
315
DataStore::const_iterator::const_iterator(const DataSet& dataset)
: m_impl(std::make_unique<Impl>(dataset)) {}

DataStore::const_iterator::const_iterator(DataSet&& dataset)
: m_impl(std::make_unique<Impl>(std::move(dataset))) {}
316

317
DataStore::const_iterator::~const_iterator() {}
318
319

DataStore::const_iterator::const_iterator(const DataStore::const_iterator& other) 
320
: m_impl(std::make_unique<Impl>(*other.m_impl)) {}
321
322

DataStore::const_iterator::const_iterator(DataStore::const_iterator&& other) 
323
: m_impl(std::move(other.m_impl)) {}
324

325
326
327
328
DataStore::const_iterator& DataStore::const_iterator::operator=(const DataStore::const_iterator& other) {
    if(&other == this) return *this;
    m_impl = std::make_unique<Impl>(*other.m_impl);
    return *this;
329
330
}

331
332
333
334
DataStore::const_iterator& DataStore::const_iterator::operator=(DataStore::const_iterator&& other) {
    if(&other == this) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
335
336
337
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++() {
338
339
340
341
342
    if(!m_impl) {
        throw Exception("Trying to increment an invalid iterator");
    }
    m_impl->m_current_dataset = m_impl->m_current_dataset.next();
    return *this;
343
344
345
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++(int) {
346
347
348
    const_iterator copy = *this;
    ++(*this);
    return copy;
349
350
351
}

const DataStore::const_iterator::reference DataStore::const_iterator::operator*() {
352
353
354
355
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return m_impl->m_current_dataset;
356
357
358
}

const DataStore::const_iterator::pointer DataStore::const_iterator::operator->() {
359
360
    if(!m_impl) return nullptr;
    return &(m_impl->m_current_dataset);
361
362
363
}

bool DataStore::const_iterator::operator==(const self_type& rhs) const {
364
365
366
367
    if(!m_impl && !rhs.m_impl)  return true;
    if(m_impl  && !rhs.m_impl)  return false;
    if(!m_impl && rhs.m_impl)   return false;
    return *m_impl == *(rhs.m_impl);
368
369
370
}

bool DataStore::const_iterator::operator!=(const self_type& rhs) const {
371
    return !(*this == rhs);
372
373
}

374
375
376
377
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::iterator implementation
////////////////////////////////////////////////////////////////////////////////////////////

378
379
380
381
382
DataStore::iterator::iterator(const DataSet& current)
: const_iterator(current) {}

DataStore::iterator::iterator(DataSet&& current)
: const_iterator(std::move(current)) {}
383

384
385
DataStore::iterator::iterator()
: const_iterator() {}
386

387
DataStore::iterator::~iterator() {}
388
389

DataStore::iterator::iterator(const DataStore::iterator& other)
390
: const_iterator(other) {}
391
392

DataStore::iterator::iterator(DataStore::iterator&& other) 
393
: const_iterator(std::move(other)) {}
394
395

DataStore::iterator& DataStore::iterator::operator=(const DataStore::iterator& other) {
396
397
398
    if(this == &other) return *this;
    m_impl = std::make_unique<Impl>(*other.m_impl);
    return *this;
399
400
401
}

DataStore::iterator& DataStore::iterator::operator=(DataStore::iterator&& other) {
402
403
404
    if(this == &other) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
405
406
407
}

DataStore::iterator::reference DataStore::iterator::operator*() {
408
    return const_cast<reference>(const_iterator::operator*());
409
410
411
}

DataStore::iterator::pointer DataStore::iterator::operator->() {
412
    return const_cast<pointer>(const_iterator::operator->());
413
414
415
416
}

}