DataStore.cpp 13.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
#include <vector>
#include <functional>
#include <iostream>
#include <yaml-cpp/yaml.h>
#include <sdskv-client.h>
#include <bake-client.h>
#include <ch-placement.h>
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
11
#include "private/DataStoreImpl.hpp"
12 13 14

namespace hepnos {

15 16 17 18
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore implementation
////////////////////////////////////////////////////////////////////////////////////////////

19
DataStore::DataStore(const std::string& configFile) 
20
: m_impl(std::make_unique<DataStore::Impl>(this)) {
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
    m_impl->init(configFile);
}

DataStore::DataStore(DataStore&& other)
: m_impl(std::move(other.m_impl)) {}

DataStore& DataStore::operator=(DataStore&& other) {
    if(&other == this) return *this;
    if(m_impl) {
        m_impl->cleanup();
    }
    m_impl = std::move(other.m_impl);
}
    
DataStore::~DataStore() {
    if(m_impl) {
        m_impl->cleanup();
    }
}

DataStore::iterator DataStore::find(const std::string& datasetName) {
42
    int ret;
43 44 45
    if(datasetName.find('/') != std::string::npos
    || datasetName.find('%') != std::string::npos) {
        throw Exception("Invalid character ('/' or '%') in dataset name");
46
    }
47
    std::vector<char> data;
48
    bool b = m_impl->load(1, "", datasetName, data);
49 50
    if(!b) {
        return m_impl->m_end;
51
    }
52
    return iterator(DataSet(this, 1, datasetName));
53 54
}

Matthieu Dorier's avatar
Matthieu Dorier committed
55 56 57 58 59
DataSet DataStore::operator[](const std::string& datasetName) const {
    auto it = find(datasetName);
    return std::move(*it);
}

60
DataStore::const_iterator DataStore::find(const std::string& datasetName) const {
61 62
    DataStore::iterator it = const_cast<DataStore*>(this)->find(datasetName);
    return it;
63 64 65
}

DataStore::iterator DataStore::begin() {
66
    DataSet ds(this, 1, "", "");
67
    ds = ds.next();
68
    if(ds.valid()) return iterator(std::move(ds));
69
    else return end();
70 71 72 73 74 75 76
}

DataStore::iterator DataStore::end() {
    return m_impl->m_end;
}

DataStore::const_iterator DataStore::cbegin() const {
77
    return const_cast<DataStore*>(this)->begin();
78 79 80
}

DataStore::const_iterator DataStore::cend() const {
81
    return const_cast<DataStore*>(this)->end();
82 83 84
}

DataStore::iterator DataStore::lower_bound(const std::string& lb) {
85 86 87 88 89 90 91 92 93
    std::string lb2 = lb;
    size_t s = lb2.size();
    lb2[s-1] -= 1; // sdskv_list_keys's start_key is exclusive
    iterator it = find(lb2);
    if(it != end()) {
        // we found something before the specified lower bound
        ++it;
        return it;
    }
94
    DataSet ds(this, 1, "", lb2);
95 96
    ds = ds.next();
    if(!ds.valid()) return end();
97
    else return iterator(std::move(ds));
98 99 100
}

DataStore::const_iterator DataStore::lower_bound(const std::string& lb) const {
101 102
    iterator it = const_cast<DataStore*>(this)->lower_bound(lb);
    return it;
103 104 105
}

DataStore::iterator DataStore::upper_bound(const std::string& ub) {
106
    DataSet ds(this, 1, "", ub);
107 108
    ds = ds.next();
    if(!ds.valid()) return end();
109
    else return iterator(std::move(ds));
110 111 112
}

DataStore::const_iterator DataStore::upper_bound(const std::string& ub) const {
113 114
    iterator it = const_cast<DataStore*>(this)->upper_bound(ub);
    return it;
115 116 117
}

DataSet DataStore::createDataSet(const std::string& name) {
118 119 120
    if(name.find('/') != std::string::npos
    || name.find('%') != std::string::npos) {
        throw Exception("Invalid character ('/' or '%') in dataset name");
121
    }
122
    m_impl->store(1, "", name, std::vector<char>());
123
    return DataSet(this, 1, "", name);
124
}
125
#if 0
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
bool DataStore::load(uint8_t level, const std::string& containerName,
        const std::string& objectName, std::vector<char>& data) const {
    int ret;
    // build full name
    std::stringstream ss;
    if(!containerName.empty())
        ss << containerName << "/";
    ss << objectName;
    // hash the name to get the provider id
    long unsigned provider_idx = 0;
    if(level != 0) {
        uint64_t h = std::hash<std::string>()(containerName);
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    } else {
        // use the complete name for final objects (level 255)
        uint64_t h = std::hash<std::string>()(ss.str());
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    }
    // make corresponding datastore entry
    DataStoreEntryPtr entry = make_datastore_entry(level, ss.str());
146 147
    auto ph = m_impl->m_sdskv_ph[provider_idx];
    auto db_id = m_impl->m_sdskv_db[provider_idx];
148 149 150 151 152 153 154 155 156
    // find the size of the value, as a way to check if the key exists
    hg_size_t vsize;
    std::cerr << "[LOG] load (level=" << (int)level
        << ", container=\"" << containerName << "\", object=\""
        << objectName << "\")" << std::endl;
    ret = sdskv_length(ph, db_id, entry->raw(), entry->length(), &vsize);
    if(ret == SDSKV_ERR_UNKNOWN_KEY) {
        return false;
    }
157
    if(ret != SDSKV_SUCCESS) {
158
        throw Exception("Error occured when calling sdskv_length");
159
    }
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
    // read the value
    data.resize(vsize);
    ret = sdskv_get(ph, db_id, entry->raw(), entry->length(), data.data(), &vsize);
    if(ret != SDSKV_SUCCESS) {
        throw Exception("Error occured when calling sdskv_get");
    }
    return true;
}

bool DataStore::store(uint8_t level, const std::string& containerName,
        const std::string& objectName, const std::vector<char>& data) {
    // build full name
    std::stringstream ss;
    if(!containerName.empty())
        ss << containerName << "/";
    ss << objectName;
    // hash the name to get the provider id
    long unsigned provider_idx = 0;
    if(level != 0) {
        uint64_t h = std::hash<std::string>()(containerName);
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    } else {
        // use the complete name for final objects (level 0)
        uint64_t h = std::hash<std::string>()(ss.str());
        ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    }
    // make corresponding datastore entry
    DataStoreEntryPtr entry = make_datastore_entry(level, ss.str());
    auto ph = m_impl->m_sdskv_ph[provider_idx];
    auto db_id = m_impl->m_sdskv_db[provider_idx];
    std::cerr << "[LOG] store (level=" << (int)level
        << ", container=\"" << containerName << "\", object=\""
        << objectName << "\")" << std::endl;
    int ret = sdskv_put(ph, db_id, entry->raw(), entry->length(), data.data(), data.size());
    if(ret != SDSKV_SUCCESS) {
        throw Exception("Could not put key/value pair in SDSKV (sdskv_put error)");
    }
    return true;
198 199
}

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
size_t DataStore::nextKeys(uint8_t level, const std::string& containerName,
        const std::string& lower,
        std::vector<std::string>& keys, size_t maxKeys) const {
    int ret;
    if(level == 0) return 0; // cannot iterate at object level
    // build full name from lower bound key
    std::stringstream ss;
    if(!containerName.empty())
        ss << containerName << "/";
    ss << lower;
    // hash the name to get the provider id
    long unsigned provider_idx = 0;
    uint64_t h = std::hash<std::string>()(containerName);
    ch_placement_find_closest(m_impl->m_chi_sdskv, h, 1, &provider_idx);
    // make an entry for the lower bound
    DataStoreEntryPtr lb_entry = make_datastore_entry(level, ss.str());
    // create data structures to receive keys
    std::vector<DataStoreEntryPtr> keys_ent;
    std::vector<void*>             keys_ptr(maxKeys);
    std::vector<hg_size_t>         keys_len(maxKeys);
    for(unsigned i=0; i < maxKeys; i++) {
        keys_ent.push_back(make_datastore_entry(level, 1024));
        keys_ptr[i] = keys_ent[i]->raw();
        keys_len[i] = sizeof(DataStoreEntry) + 1024;
    }
    // get provider and database
    auto ph = m_impl->m_sdskv_ph[provider_idx];
    auto db_id = m_impl->m_sdskv_db[provider_idx];
    // issue an sdskv_list_keys
    hg_size_t max_keys = maxKeys;
    std::cerr << "[LOG] list keys (level=" << (int)level
        << ", container=\"" << containerName << "\", greaterthan=\""
        << lower << "\")" << std::endl;
    ret = sdskv_list_keys(ph, db_id, lb_entry->raw(), lb_entry->length(),
                          keys_ptr.data(), keys_len.data(), &max_keys);
    if(ret != HG_SUCCESS) {
        throw Exception("Error occured when calling sdskv_list_keys");
    }
    unsigned i = max_keys - 1;
    if(max_keys == 0) return 0;
    // remove keys that don't have the same level or the same prefix
    std::string prefix = containerName + "/";
    keys.resize(0);
    for(unsigned i = 0; i < max_keys; i++) {
        if(keys_ent[i]->m_level != level) {
            max_keys = i;
            break;
        }
        if(!containerName.empty()) {
            size_t lenpre = prefix.size();
            size_t lenstr = strlen(keys_ent[i]->m_fullname);
            if(lenstr < lenpre) {
                max_keys = i;
                break;
            }
            if(strncmp(prefix.c_str(), keys_ent[i]->m_fullname, lenpre) != 0) {
                max_keys = i;
                break;
            }
        }
        keys.push_back(keys_ent[i]->m_fullname);
    }
    // set the resulting keys
    return max_keys;
}
265
#endif
266 267 268 269
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////

270 271 272 273
class DataStore::const_iterator::Impl {
    public:
        DataSet    m_current_dataset;

274 275
        Impl()
        : m_current_dataset()
276
        {}
277

278 279 280 281 282 283
        Impl(const DataSet& dataset)
        : m_current_dataset(dataset)
        {}

        Impl(DataSet&& dataset)
        : m_current_dataset(std::move(dataset))
284
        {}
285 286

        Impl(const Impl& other)
287
        : m_current_dataset(other.m_current_dataset) 
288 289 290
        {}

        bool operator==(const Impl& other) const {
291
            return m_current_dataset == other.m_current_dataset;
292
        }
293 294
};

295 296 297
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
298

299 300
DataStore::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}
301

302 303 304 305 306
DataStore::const_iterator::const_iterator(const DataSet& dataset)
: m_impl(std::make_unique<Impl>(dataset)) {}

DataStore::const_iterator::const_iterator(DataSet&& dataset)
: m_impl(std::make_unique<Impl>(std::move(dataset))) {}
307

308
DataStore::const_iterator::~const_iterator() {}
309 310

DataStore::const_iterator::const_iterator(const DataStore::const_iterator& other) 
311
: m_impl(std::make_unique<Impl>(*other.m_impl)) {}
312 313

DataStore::const_iterator::const_iterator(DataStore::const_iterator&& other) 
314
: m_impl(std::move(other.m_impl)) {}
315

316 317 318 319
DataStore::const_iterator& DataStore::const_iterator::operator=(const DataStore::const_iterator& other) {
    if(&other == this) return *this;
    m_impl = std::make_unique<Impl>(*other.m_impl);
    return *this;
320 321
}

322 323 324 325
DataStore::const_iterator& DataStore::const_iterator::operator=(DataStore::const_iterator&& other) {
    if(&other == this) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
326 327 328
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++() {
329 330 331 332 333
    if(!m_impl) {
        throw Exception("Trying to increment an invalid iterator");
    }
    m_impl->m_current_dataset = m_impl->m_current_dataset.next();
    return *this;
334 335 336
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++(int) {
337 338 339
    const_iterator copy = *this;
    ++(*this);
    return copy;
340 341 342
}

const DataStore::const_iterator::reference DataStore::const_iterator::operator*() {
343 344 345 346
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return m_impl->m_current_dataset;
347 348 349
}

const DataStore::const_iterator::pointer DataStore::const_iterator::operator->() {
350 351
    if(!m_impl) return nullptr;
    return &(m_impl->m_current_dataset);
352 353 354
}

bool DataStore::const_iterator::operator==(const self_type& rhs) const {
355 356 357 358
    if(!m_impl && !rhs.m_impl)  return true;
    if(m_impl  && !rhs.m_impl)  return false;
    if(!m_impl && rhs.m_impl)   return false;
    return *m_impl == *(rhs.m_impl);
359 360 361
}

bool DataStore::const_iterator::operator!=(const self_type& rhs) const {
362
    return !(*this == rhs);
363 364
}

365 366 367 368
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::iterator implementation
////////////////////////////////////////////////////////////////////////////////////////////

369 370 371 372 373
DataStore::iterator::iterator(const DataSet& current)
: const_iterator(current) {}

DataStore::iterator::iterator(DataSet&& current)
: const_iterator(std::move(current)) {}
374

375 376
DataStore::iterator::iterator()
: const_iterator() {}
377

378
DataStore::iterator::~iterator() {}
379 380

DataStore::iterator::iterator(const DataStore::iterator& other)
381
: const_iterator(other) {}
382 383

DataStore::iterator::iterator(DataStore::iterator&& other) 
384
: const_iterator(std::move(other)) {}
385 386

DataStore::iterator& DataStore::iterator::operator=(const DataStore::iterator& other) {
387 388 389
    if(this == &other) return *this;
    m_impl = std::make_unique<Impl>(*other.m_impl);
    return *this;
390 391 392
}

DataStore::iterator& DataStore::iterator::operator=(DataStore::iterator&& other) {
393 394 395
    if(this == &other) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
396 397 398
}

DataStore::iterator::reference DataStore::iterator::operator*() {
399
    return const_cast<reference>(const_iterator::operator*());
400 401 402
}

DataStore::iterator::pointer DataStore::iterator::operator->() {
403
    return const_cast<pointer>(const_iterator::operator->());
404 405 406 407
}

}