DataStore.cpp 11.6 KB
Newer Older
Matthieu Dorier's avatar
Matthieu Dorier committed
1 2 3 4 5
/*
 * (C) 2018 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */
6 7 8 9 10 11
#include <vector>
#include <functional>
#include <iostream>
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
12
#include "hepnos/WriteBatch.hpp"
13
#include "DataSetImpl.hpp"
14 15
#include "DataStoreImpl.hpp"
#include "WriteBatchImpl.hpp"
16 17 18

namespace hepnos {

19 20 21 22
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore implementation
////////////////////////////////////////////////////////////////////////////////////////////

23 24 25 26 27 28 29 30 31 32 33
DataStore::DataStore(std::shared_ptr<Impl>&& impl)
: m_impl(std::move(impl)) {}

DataStore::DataStore(const std::shared_ptr<Impl>& impl)
: m_impl(impl) {}

bool DataStore::valid() const {
    return m_impl != nullptr;
}

DataStore DataStore::connect() { 
34 35 36 37
    char* file = getenv("HEPNOS_CONFIG_FILE");
    if(file == nullptr) 
        throw Exception("HEPNOS_CONFIG_FILE environment variable not set");
    std::string configFile(file);
38 39 40
    auto impl = std::make_shared<Impl>();
    impl->init(configFile);
    return DataStore(std::move(impl));
41 42
}

43 44 45 46
DataStore DataStore::connect(const std::string& configFile) {
    auto impl = std::make_shared<Impl>();
    impl->init(configFile);
    return DataStore(std::move(impl));
47 48
}

49
DataStore::iterator DataStore::find(const std::string& datasetPath) {
50 51 52 53
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }

54
    int ret;
55 56 57

    if(datasetPath.find('%') != std::string::npos) {
        throw Exception("Invalid character ('%') in dataset name");
58
    }
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73

    size_t slash_count = std::count(datasetPath.begin(), datasetPath.end(), '/');
    size_t level = 1 + slash_count;
    std::string containerName;
    std::string datasetName;

    if(slash_count == 0) {
        datasetName = datasetPath;
        containerName = "";
    } else {
        size_t c = datasetPath.find_last_of('/');
        containerName = datasetPath.substr(0,c);
        datasetName   = datasetPath.substr(c+1);
    }

74
    std::string data;
75
    bool b = m_impl->load(level, containerName, datasetName, data);
76 77
    if(!b) {
        return m_impl->m_end;
78
    }
79 80 81 82
    return iterator(
            DataSet(
                std::make_shared<DataSet::Impl>(
                    m_impl, level, std::make_shared<std::string>(containerName), datasetName)));
83 84
}

Matthieu Dorier's avatar
Matthieu Dorier committed
85 86
DataSet DataStore::operator[](const std::string& datasetName) const {
    auto it = find(datasetName);
87 88
    if(!it->valid())
        throw Exception("Requested DataSet does not exist");
Matthieu Dorier's avatar
Matthieu Dorier committed
89 90 91
    return std::move(*it);
}

92
DataStore::const_iterator DataStore::find(const std::string& datasetName) const {
93 94
    DataStore::iterator it = const_cast<DataStore*>(this)->find(datasetName);
    return it;
95 96 97
}

DataStore::iterator DataStore::begin() {
98 99 100
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
101 102 103
    DataSet ds(
            std::make_shared<DataSet::Impl>(
                m_impl, 1, std::make_shared<std::string>(""), ""));
104
    ds = ds.next();
105
    if(ds.valid()) return iterator(std::move(ds));
106
    else return end();
107 108
}

109 110 111 112 113
DataStore::const_iterator DataStore::begin() const {
    iterator it = const_cast<DataStore*>(this)->begin();
    return const_iterator(std::move(it));
}

114
DataStore::iterator DataStore::end() {
115 116 117
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
118 119 120
    return m_impl->m_end;
}

121
DataStore::const_iterator DataStore::end() const {
122 123 124
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
125 126 127
    return m_impl->m_end;
}

128
DataStore::const_iterator DataStore::cbegin() const {
129
    return const_cast<DataStore*>(this)->begin();
130 131 132
}

DataStore::const_iterator DataStore::cend() const {
133
    return const_cast<DataStore*>(this)->end();
134 135 136
}

DataStore::iterator DataStore::lower_bound(const std::string& lb) {
137 138 139 140 141 142 143 144 145
    std::string lb2 = lb;
    size_t s = lb2.size();
    lb2[s-1] -= 1; // sdskv_list_keys's start_key is exclusive
    iterator it = find(lb2);
    if(it != end()) {
        // we found something before the specified lower bound
        ++it;
        return it;
    }
146 147 148
    DataSet ds(
            std::make_shared<DataSet::Impl>(
                m_impl, 1, std::make_shared<std::string>(""), lb2));
149 150
    ds = ds.next();
    if(!ds.valid()) return end();
151
    else return iterator(std::move(ds));
152 153 154
}

DataStore::const_iterator DataStore::lower_bound(const std::string& lb) const {
155 156
    iterator it = const_cast<DataStore*>(this)->lower_bound(lb);
    return it;
157 158 159
}

DataStore::iterator DataStore::upper_bound(const std::string& ub) {
160 161 162
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
163 164 165
    DataSet ds(
            std::make_shared<DataSet::Impl>(
                m_impl, 1, std::make_shared<std::string>(""), ub));
166 167
    ds = ds.next();
    if(!ds.valid()) return end();
168
    else return iterator(std::move(ds));
169 170 171
}

DataStore::const_iterator DataStore::upper_bound(const std::string& ub) const {
172 173
    iterator it = const_cast<DataStore*>(this)->upper_bound(ub);
    return it;
174 175 176
}

DataSet DataStore::createDataSet(const std::string& name) {
177 178 179
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
180 181 182
    if(name.find('/') != std::string::npos
    || name.find('%') != std::string::npos) {
        throw Exception("Invalid character ('/' or '%') in dataset name");
183
    }
184
    m_impl->store(1, "", name);
185 186 187
    return DataSet(
            std::make_shared<DataSet::Impl>(
                m_impl, 1, std::make_shared<std::string>(""), name));
188 189 190 191 192 193 194 195 196 197
}

DataSet DataStore::createDataSet(WriteBatch& batch, const std::string& name) {
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
    if(name.find('/') != std::string::npos
    || name.find('%') != std::string::npos) {
        throw Exception("Invalid character ('/' or '%') in dataset name");
    }
198
    batch.m_impl->store(1, "", name);
199 200 201
    return DataSet(
            std::make_shared<DataSet::Impl>(
                m_impl, 1, std::make_shared<std::string>(""), name));
202 203
}

204
void DataStore::shutdown() {
205 206 207
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
208
    for(auto addr : m_impl->m_addrs) {
Matthieu Dorier's avatar
Matthieu Dorier committed
209
        margo_shutdown_remote_instance(m_impl->m_mid, addr.second);
210 211 212
    }
}

213
bool DataStore::loadRawProduct(const ProductID& productID, std::string& buffer) {
214 215 216
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
217 218 219
    return m_impl->load(productID.m_level, productID.m_containerName, productID.m_objectName, buffer);
}

220 221 222 223 224 225 226
bool DataStore::loadRawProduct(const ProductID& productID, char* data, size_t* size) {
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
    return m_impl->load(productID.m_level, productID.m_containerName, productID.m_objectName, data, size);
}

227 228 229 230
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////

231 232 233 234
class DataStore::const_iterator::Impl {
    public:
        DataSet    m_current_dataset;

235 236
        Impl()
        : m_current_dataset()
237
        {}
238

239 240 241 242 243 244
        Impl(const DataSet& dataset)
        : m_current_dataset(dataset)
        {}

        Impl(DataSet&& dataset)
        : m_current_dataset(std::move(dataset))
245
        {}
246 247

        Impl(const Impl& other)
248
        : m_current_dataset(other.m_current_dataset) 
249 250 251
        {}

        bool operator==(const Impl& other) const {
252
            return m_current_dataset == other.m_current_dataset;
253
        }
254 255
};

256 257 258
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
259

260 261
DataStore::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}
262

263 264 265 266 267
DataStore::const_iterator::const_iterator(const DataSet& dataset)
: m_impl(std::make_unique<Impl>(dataset)) {}

DataStore::const_iterator::const_iterator(DataSet&& dataset)
: m_impl(std::make_unique<Impl>(std::move(dataset))) {}
268

269
DataStore::const_iterator::~const_iterator() {}
270

271 272 273 274 275
DataStore::const_iterator::const_iterator(const DataStore::const_iterator& other) {
    if(other.m_impl) {
        m_impl = std::make_unique<Impl>(*other.m_impl);
    }
}
276 277

DataStore::const_iterator::const_iterator(DataStore::const_iterator&& other) 
278
: m_impl(std::move(other.m_impl)) {}
279

280 281
DataStore::const_iterator& DataStore::const_iterator::operator=(const DataStore::const_iterator& other) {
    if(&other == this) return *this;
282 283 284 285
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
286
    return *this;
287 288
}

289 290 291 292
DataStore::const_iterator& DataStore::const_iterator::operator=(DataStore::const_iterator&& other) {
    if(&other == this) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
293 294 295
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++() {
296 297 298 299 300
    if(!m_impl) {
        throw Exception("Trying to increment an invalid iterator");
    }
    m_impl->m_current_dataset = m_impl->m_current_dataset.next();
    return *this;
301 302 303
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++(int) {
304 305 306
    const_iterator copy = *this;
    ++(*this);
    return copy;
307 308 309
}

const DataStore::const_iterator::reference DataStore::const_iterator::operator*() {
310 311 312 313
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return m_impl->m_current_dataset;
314 315 316
}

const DataStore::const_iterator::pointer DataStore::const_iterator::operator->() {
317 318 319
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
320
    return &(m_impl->m_current_dataset);
321 322 323
}

bool DataStore::const_iterator::operator==(const self_type& rhs) const {
324 325 326 327
    if(!m_impl && !rhs.m_impl)  return true;
    if(m_impl  && !rhs.m_impl)  return false;
    if(!m_impl && rhs.m_impl)   return false;
    return *m_impl == *(rhs.m_impl);
328 329 330
}

bool DataStore::const_iterator::operator!=(const self_type& rhs) const {
331
    return !(*this == rhs);
332 333
}

334 335 336 337
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::iterator implementation
////////////////////////////////////////////////////////////////////////////////////////////

338 339 340 341 342
DataStore::iterator::iterator(const DataSet& current)
: const_iterator(current) {}

DataStore::iterator::iterator(DataSet&& current)
: const_iterator(std::move(current)) {}
343

344 345
DataStore::iterator::iterator()
: const_iterator() {}
346

347
DataStore::iterator::~iterator() {}
348 349

DataStore::iterator::iterator(const DataStore::iterator& other)
350
: const_iterator(other) {}
351 352

DataStore::iterator::iterator(DataStore::iterator&& other) 
353
: const_iterator(std::move(other)) {}
354 355

DataStore::iterator& DataStore::iterator::operator=(const DataStore::iterator& other) {
356
    if(this == &other) return *this;
357 358 359 360
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
361
    return *this;
362 363 364
}

DataStore::iterator& DataStore::iterator::operator=(DataStore::iterator&& other) {
365 366 367
    if(this == &other) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
368 369 370
}

DataStore::iterator::reference DataStore::iterator::operator*() {
371
    return const_cast<reference>(const_iterator::operator*());
372 373 374
}

DataStore::iterator::pointer DataStore::iterator::operator->() {
375
    return const_cast<pointer>(const_iterator::operator->());
376 377 378 379
}

}